{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14661932,"sourceType":"datasetVersion","datasetId":9366631}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Version B: Length Penalty 1.3\n## Testing LOWER length penalty (1.5 → 1.3)\n\nSince 1.8 made it worse (34.8), let's try going the other direction.\nMaybe shorter, more concise translations score better.","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\nos.environ[\"TORCH_CUDNN_V8_API_ENABLED\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\nimport re, logging, warnings\nfrom pathlib import Path\nfrom typing import List\nfrom dataclasses import dataclass\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport random\n\nwarnings.filterwarnings(\"ignore\")\nprint(f\"PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-06T05:41:26.492782Z","iopub.execute_input":"2026-02-06T05:41:26.493038Z","iopub.status.idle":"2026-02-06T05:41:39.699375Z","shell.execute_reply.started":"2026-02-06T05:41:26.49301Z","shell.execute_reply":"2026-02-06T05:41:39.69872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@dataclass\nclass UltraConfig:\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    model_path: str = \"/kaggle/input/final-byt5/byt5-akkadian-optimized-34x\"\n    output_dir: str = \"/kaggle/working/\"\n    max_length: int = 512\n    batch_size: int = 8\n    num_workers: int = 4\n    num_beams: int = 8\n    max_new_tokens: int = 512\n    length_penalty: float = 1.3  # TESTING LOWER (baseline was 1.5)\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 0\n    use_mixed_precision: bool = True\n    use_better_transformer: bool = True\n    use_bucket_batching: bool = True\n    use_vectorized_postproc: bool = True\n    use_adaptive_beams: bool = True\n    aggressive_postprocessing: bool = True\n    num_buckets: int = 4\n    \n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\nconfig = UltraConfig()\nprint(f\"Config: LP={config.length_penalty} (testing lower than 1.5)\")\n\ndef setup_logging(output_dir):\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n    for h in logging.root.handlers[:]: logging.root.removeHandler(h)\n    logging.basicConfig(level=logging.INFO, format=\"%(message)s\", handlers=[logging.StreamHandler()])\n    return logging.getLogger(__name__)\n\nlogger = setup_logging(config.output_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-06T05:41:39.700885Z","iopub.execute_input":"2026-02-06T05:41:39.701323Z","iopub.status.idle":"2026-02-06T05:41:39.710238Z","shell.execute_reply.started":"2026-02-06T05:41:39.701301Z","shell.execute_reply":"2026-02-06T05:41:39.709615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [Same preprocessing/postprocessing classes as before - copying from baseline]\n\nclass OptimizedPreprocessor:\n    def __init__(self):\n        self.patterns = {\"big_gap\": re.compile(r\"(\\\\.{3,}|…+|——)\"), \"small_gap\": re.compile(r\"(xx+|\\\\s+x\\\\s+)\")}\n    def preprocess_batch(self, texts):\n        s = pd.Series(texts).fillna(\"\").astype(str)\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n        return s.tolist()\n\nclass VectorizedPostprocessor:\n    def __init__(self, aggressive=True):\n        self.aggressive = aggressive\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\\\[x\\\\]|\\\\(x\\\\)|\\\\bx\\\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\\\.{3,}|…|\\\\[\\\\.+\\\\])\"),\n            \"annotations\": re.compile(r\"\\\\((fem|plur|pl|sing|singular|plural|\\\\?|!)\\\\..\\\\s*\\\\w*\\\\)\", re.I),\n            \"repeated_words\": re.compile(r\"\\\\b(\\\\w+)(?:\\\\s+\\\\1\\\\b)+\"),\n            \"whitespace\": re.compile(r\"\\\\s+\"),\n            \"punct_space\": re.compile(r\"\\\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\\\1+\"),\n        }\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n        self.forbidden_trans = str.maketrans(\"\", \"\", '!?()\"——<>⌈⌋⌊[]+ʾ/;')\n    \n    def postprocess_batch(self, translations):\n        s = pd.Series(translations)\n        valid_mask = s.apply(lambda x: isinstance(x, str) and bool(x.strip()))\n        s.loc[~valid_mask] = \"\"\n        s = s.str.translate(self.special_chars_trans).str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True).str.strip()\n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n            s = s.str.replace(\"<gap>\", \"\\\\x00G\\\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\\\x00B\\\\x00\", regex=False)\n            s = s.str.translate(self.forbidden_trans)\n            s = s.str.replace(\"\\\\x00G\\\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\\\x00B\\\\x00\", \" <big_gap> \", regex=False)\n            s = s.str.replace(r\"(\\\\d+)\\\\.5\\\\b\", r\"\\\\1½\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.5\\\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\\\d+)\\\\.25\\\\b\", r\"\\\\1¼\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.25\\\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\\\d+)\\\\.75\\\\b\", r\"\\\\1¾\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.75\\\\b\", \"¾\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\\\1\", regex=True)\n            for n in range(4, 1, -1):\n                s = s.str.replace(r\"\\\\b((?:\\\\w+\\\\s+){\" + str(n-1) + r\"}\\\\w+)(?:\\\\s+\\\\1\\\\b)+\", r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n        return s.tolist()\n\nclass BucketBatchSampler(Sampler):\n    def __init__(self, dataset, batch_size, num_buckets=4, shuffle=False):\n        self.dataset, self.batch_size, self.shuffle = dataset, batch_size, shuffle\n        lengths = [len(dataset[i][1].split()) for i in range(len(dataset))]\n        sorted_idx = sorted(range(len(lengths)), key=lambda i: lengths[i])\n        bucket_size = max(1, len(sorted_idx) // num_buckets)\n        self.buckets = [sorted_idx[i*bucket_size:(i+1)*bucket_size if i<num_buckets-1 else None] for i in range(num_buckets)]\n    def __iter__(self):\n        for bucket in self.buckets:\n            if self.shuffle: random.shuffle(bucket)\n            for i in range(0, len(bucket), self.batch_size): yield bucket[i:i+self.batch_size]\n    def __len__(self):\n        return sum((len(b) + self.batch_size - 1) // self.batch_size for b in self.buckets)\n\nclass AkkadianDataset(Dataset):\n    def __init__(self, df, preprocessor):\n        self.sample_ids = df[\"id\"].tolist()\n        preprocessed = preprocessor.preprocess_batch(df[\"transliteration\"].tolist())\n        self.input_texts = [\"translate Akkadian to English: \" + text for text in preprocessed]\n    def __len__(self): return len(self.sample_ids)\n    def __getitem__(self, idx): return self.sample_ids[idx], self.input_texts[idx]\n\nclass UltraInferenceEngine:\n    def __init__(self, config):\n        self.config = config\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(config.aggressive_postprocessing)\n        self.results = []\n        logger.info(f\"Loading model\")\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(config.model_path).to(config.device).eval()\n        self.tokenizer = AutoTokenizer.from_pretrained(config.model_path)\n        if config.use_better_transformer and torch.cuda.is_available():\n            try:\n                from optimum.bettertransformer import BetterTransformer\n                self.model = BetterTransformer.transform(self.model)\n            except: pass\n    \n    def _collate_fn(self, batch):\n        ids, texts = [s[0] for s in batch], [s[1] for s in batch]\n        return ids, self.tokenizer(texts, max_length=self.config.max_length, padding=True, truncation=True, return_tensors=\"pt\")\n    \n    def _get_adaptive_beam_size(self, attention_mask):\n        if not self.config.use_adaptive_beams: return self.config.num_beams\n        lengths = attention_mask.sum(dim=1)\n        return max(4, self.config.num_beams // 2) if lengths[0] < 100 else self.config.num_beams\n    \n    def run_inference(self, test_df):\n        dataset = AkkadianDataset(test_df, self.preprocessor)\n        if self.config.use_bucket_batching:\n            sampler = BucketBatchSampler(dataset, self.config.batch_size, self.config.num_buckets)\n            dataloader = DataLoader(dataset, batch_sampler=sampler, num_workers=self.config.num_workers,\n                                   collate_fn=self._collate_fn, pin_memory=True, prefetch_factor=2,\n                                   persistent_workers=self.config.num_workers > 0)\n        else:\n            dataloader = DataLoader(dataset, batch_size=self.config.batch_size, shuffle=False,\n                                   num_workers=self.config.num_workers, collate_fn=self._collate_fn,\n                                   pin_memory=True, prefetch_factor=2, persistent_workers=self.config.num_workers > 0)\n        \n        gen_config = {\"max_new_tokens\": self.config.max_new_tokens, \"length_penalty\": self.config.length_penalty,\n                     \"early_stopping\": self.config.early_stopping, \"use_cache\": True}\n        if self.config.no_repeat_ngram_size > 0: gen_config[\"no_repeat_ngram_size\"] = self.config.no_repeat_ngram_size\n        \n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(tqdm(dataloader, desc=\"Translating\")):\n                try:\n                    input_ids, attention_mask = tokenized.input_ids.to(self.config.device), tokenized.attention_mask.to(self.config.device)\n                    beam_size = self._get_adaptive_beam_size(attention_mask)\n                    cfg = {**gen_config, \"num_beams\": beam_size}\n                    \n                    with autocast() if self.config.use_mixed_precision else torch.no_grad():\n                        outputs = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **cfg)\n                    \n                    translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n                    cleaned = self.postprocessor.postprocess_batch(translations) if self.config.use_vectorized_postproc else \\\n                             [self.postprocessor.postprocess_batch([t])[0] for t in translations]\n                    self.results.extend(zip(batch_ids, cleaned))\n                    \n                    if torch.cuda.is_available() and batch_idx % 10 == 0: torch.cuda.empty_cache()\n                except Exception as e:\n                    logger.error(f\"Batch {batch_idx} error: {e}\")\n                    self.results.extend([(bid, \"\") for bid in batch_ids])\n        \n        return pd.DataFrame(self.results, columns=[\"id\", \"translation\"])\n\nprint(\"Ready\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-06T05:41:39.711385Z","iopub.execute_input":"2026-02-06T05:41:39.711625Z","iopub.status.idle":"2026-02-06T05:41:39.848326Z","shell.execute_reply.started":"2026-02-06T05:41:39.711602Z","shell.execute_reply":"2026-02-06T05:41:39.847618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(config.test_data_path)\nlogger.info(f\"Loaded {len(test_df)} samples\")\n\nengine = UltraInferenceEngine(config)\nresults_df = engine.run_inference(test_df)\n\nresults_df.to_csv(Path(config.output_dir) / \"submission.csv\", index=False)\nprint(f\"\\\\n✓ Done! LP=1.3 (baseline was 1.5, testing if lower is better)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-06T05:41:39.849301Z","iopub.execute_input":"2026-02-06T05:41:39.849609Z","iopub.status.idle":"2026-02-06T05:42:28.474259Z","shell.execute_reply.started":"2026-02-06T05:41:39.849581Z","shell.execute_reply":"2026-02-06T05:42:28.473446Z"}},"outputs":[],"execution_count":null}]}