{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14190587,"sourceType":"datasetVersion","datasetId":9048186},{"sourceId":14661932,"sourceType":"datasetVersion","datasetId":9366631},{"sourceId":691084,"sourceType":"modelInstanceVersion","modelInstanceId":523956,"modelId":537969},{"sourceId":746352,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":569858,"modelId":582145}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Challenge: Akkadian → English Translation\n\n\n","metadata":{}},{"cell_type":"code","source":"# Cell 1\nimport os\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\nos.environ[\"TORCH_CUDNN_V8_API_ENABLED\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\nimport re\nimport logging\nimport warnings\nfrom pathlib import Path\nfrom typing import List\nfrom dataclasses import dataclass\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport random\n\nwarnings.filterwarnings(\"ignore\")\n\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:35.189354Z","iopub.execute_input":"2026-02-09T14:55:35.190059Z","iopub.status.idle":"2026-02-09T14:55:47.762474Z","shell.execute_reply.started":"2026-02-09T14:55:35.190028Z","shell.execute_reply":"2026-02-09T14:55:47.761725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2\n@dataclass\nclass UltraConfig:\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    model_path: str = \"/kaggle/input/nllb-200-akadian-to-english/transformers/default/1/byt5-base-akkadian_gap3\"\n    output_dir: str = \"/kaggle/working/\"\n    \n    max_length: int = 512\n    batch_size: int = 8\n    num_workers: int = 4\n    \n    num_beams: int = 8\n    max_new_tokens: int = 512\n    length_penalty: float = 1.5\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 0\n    \n    use_mixed_precision: bool = True\n    use_better_transformer: bool = True\n    use_bucket_batching: bool = True\n    use_vectorized_postproc: bool = True\n    use_adaptive_beams: bool = True\n    use_auto_batch_size: bool = False\n    \n    aggressive_postprocessing: bool = True\n    checkpoint_freq: int = 100\n    num_buckets: int = 4\n    \n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\ncfg = UltraConfig()\nprint(f\"\\nConfig: beams={cfg.num_beams}, length_penalty={cfg.length_penalty}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:52.033242Z","iopub.execute_input":"2026-02-09T14:55:52.034381Z","iopub.status.idle":"2026-02-09T14:55:52.042516Z","shell.execute_reply.started":"2026-02-09T14:55:52.034335Z","shell.execute_reply":"2026-02-09T14:55:52.041811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3\ndef setup_logging(output_dir: str = \"./outputs\"):\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n    for handler in logging.root.handlers[:]:\n        logging.root.removeHandler(handler)\n    logging.basicConfig(\n        level=logging.INFO,\n        format=\"%(asctime)s - %(levelname)s - %(message)s\",\n        handlers=[logging.StreamHandler()],\n    )\n    return logging.getLogger(__name__)\n\nlog = setup_logging(cfg.output_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:53.668998Z","iopub.execute_input":"2026-02-09T14:55:53.669511Z","iopub.status.idle":"2026-02-09T14:55:53.674640Z","shell.execute_reply.started":"2026-02-09T14:55:53.669484Z","shell.execute_reply":"2026-02-09T14:55:53.673984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4\nclass OptimizedPreprocessor:\n    def __init__(self):\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|——)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\"),\n        }\n    \n    def preprocess_input_text(self, text: str) -> str:\n        if pd.isna(text):\n            return \"\"\n        text = str(text)\n        text = self.patterns[\"big_gap\"].sub(\"<big_gap>\", text)\n        text = self.patterns[\"small_gap\"].sub(\"<gap>\", text)\n        return text\n    \n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        s = pd.Series(texts).fillna(\"\")\n        s = s.astype(str)\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n        return s.tolist()\n\ntext_preproc = OptimizedPreprocessor()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:54.812535Z","iopub.execute_input":"2026-02-09T14:55:54.813307Z","iopub.status.idle":"2026-02-09T14:55:54.819403Z","shell.execute_reply.started":"2026-02-09T14:55:54.813279Z","shell.execute_reply":"2026-02-09T14:55:54.818647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5\nclass VectorizedPostprocessor:\n    def __init__(self, aggressive: bool = True):\n        self.aggressive = aggressive\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\.{3,}|…|\\[\\.+\\])\"),\n            \"annotations\": re.compile(r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..\\s*\\w*\\)\", re.I),\n            \"repeated_words\": re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\"),\n            \"whitespace\": re.compile(r\"\\s+\"),\n            \"punct_space\": re.compile(r\"\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\1+\"),\n        }\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n        self.forbidden_chars = '!?()\"——<>⌈⌋⌊[]+ʾ/;'\n        self.forbidden_trans = str.maketrans(\"\", \"\", self.forbidden_chars)\n    \n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        s = pd.Series(translations)\n        valid_mask = s.apply(lambda x: isinstance(x, str) and x.strip())\n        if not valid_mask.all():\n            s[~valid_mask] = \"\"\n        \n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n        s = s.str.strip()\n        \n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n            \n            s = s.str.replace(\"<gap>\", \"\\x00GAP\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\x00BIG\\x00\", regex=False)\n            s = s.str.translate(self.forbidden_trans)\n            s = s.str.replace(\"\\x00GAP\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\x00BIG\\x00\", \" <big_gap> \", regex=False)\n            \n            s = s.str.replace(r\"(\\d+)\\.5\\b\", r\"\\1½\", regex=True)\n            s = s.str.replace(r\"\\b0\\.5\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.25\\b\", r\"\\1¼\", regex=True)\n            s = s.str.replace(r\"\\b0\\.25\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.75\\b\", r\"\\1¾\", regex=True)\n            s = s.str.replace(r\"\\b0\\.75\\b\", \"¾\", regex=True)\n            \n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\1\", regex=True)\n            for n in range(4, 1, -1):\n                pattern = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                s = s.str.replace(pattern, r\"\\1\", regex=True)\n            \n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n        \n        return s.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:55.952461Z","iopub.execute_input":"2026-02-09T14:55:55.953219Z","iopub.status.idle":"2026-02-09T14:55:55.964243Z","shell.execute_reply.started":"2026-02-09T14:55:55.953190Z","shell.execute_reply":"2026-02-09T14:55:55.963546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 6\nclass BucketBatchSampler(Sampler):\n    def __init__(self, dataset, batch_size, num_buckets=4, shuffle=False):\n        self.dataset = dataset\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        \n        lengths = [len(dataset[i][1].split()) for i in range(len(dataset))]\n        sorted_indices = sorted(range(len(lengths)), key=lambda i: lengths[i])\n        \n        bucket_size = max(1, len(sorted_indices) // num_buckets)\n        self.buckets = []\n        for i in range(num_buckets):\n            start = i * bucket_size\n            end = None if i == num_buckets - 1 else (i + 1) * bucket_size\n            self.buckets.append(sorted_indices[start:end])\n    \n    def __iter__(self):\n        for bucket in self.buckets:\n            if self.shuffle:\n                random.shuffle(bucket)\n            for i in range(0, len(bucket), self.batch_size):\n                yield bucket[i:i+self.batch_size]\n    \n    def __len__(self):\n        return sum((len(b) + self.batch_size - 1) // self.batch_size for b in self.buckets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:56.386559Z","iopub.execute_input":"2026-02-09T14:55:56.387298Z","iopub.status.idle":"2026-02-09T14:55:56.393930Z","shell.execute_reply.started":"2026-02-09T14:55:56.387273Z","shell.execute_reply":"2026-02-09T14:55:56.393215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Cell 7\nclass AkkadianDataset(Dataset):\n    def __init__(self, dataframe: pd.DataFrame, preprocessor: OptimizedPreprocessor):\n        self.sample_ids = dataframe[\"id\"].tolist()\n        raw_texts = dataframe[\"transliteration\"].tolist()\n        preprocessed = preprocessor.preprocess_batch(raw_texts)\n        self.input_texts = [\"translate Akkadian to English: \" + text for text in preprocessed]\n        log.info(f\"Dataset created with {len(self.sample_ids)} samples\")\n    \n    def __len__(self):\n        return len(self.sample_ids)\n    \n    def __getitem__(self, index: int):\n        return self.sample_ids[index], self.input_texts[index]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:57.803359Z","iopub.execute_input":"2026-02-09T14:55:57.803678Z","iopub.status.idle":"2026-02-09T14:55:57.808931Z","shell.execute_reply.started":"2026-02-09T14:55:57.803655Z","shell.execute_reply":"2026-02-09T14:55:57.808293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 8\nclass UltraInferenceEngine:\n    def __init__(self, config: UltraConfig):\n        self.config = config\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(aggressive=config.aggressive_postprocessing)\n        self.results = []\n        self._load_model()\n    \n    def _load_model(self):\n        log.info(f\"Loading model from {self.config.model_path}\")\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path).to(self.config.device).eval()\n        self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_path)\n        num_params = sum(p.numel() for p in self.model.parameters())\n        log.info(f\"Model loaded: {num_params:,} parameters\")\n        \n        if self.config.use_better_transformer and torch.cuda.is_available():\n            try:\n                from optimum.bettertransformer import BetterTransformer\n                log.info(\"Applying BetterTransformer...\")\n                self.model = BetterTransformer.transform(self.model)\n                log.info(\"BetterTransformer applied\")\n            except ImportError:\n                log.warning(\"optimum not installed, skipping BetterTransformer\")\n            except Exception as e:\n                log.warning(f\"BetterTransformer failed: {e}\")\n    \n    def _collate_fn(self, batch_samples):\n        batch_ids = [s[0] for s in batch_samples]\n        batch_texts = [s[1] for s in batch_samples]\n        tokenized = self.tokenizer(batch_texts, max_length=self.config.max_length, \n                                   padding=True, truncation=True, return_tensors=\"pt\")\n        return batch_ids, tokenized\n    \n    def _get_adaptive_beam_size(self, input_ids, attention_mask):\n        if not self.config.use_adaptive_beams:\n            return self.config.num_beams\n        lengths = attention_mask.sum(dim=1)\n        beam_sizes = torch.where(\n            lengths < 100,\n            torch.tensor(max(4, self.config.num_beams // 2)),\n            torch.tensor(self.config.num_beams),\n        )\n        return beam_sizes[0].item()\n    \n    def run_inference(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        log.info(\"Starting inference\")\n        dataset = AkkadianDataset(test_df, self.preprocessor)\n        \n        if self.config.use_bucket_batching:\n            batch_sampler = BucketBatchSampler(dataset, self.config.batch_size, num_buckets=self.config.num_buckets)\n            data_loader = DataLoader(\n                dataset, batch_sampler=batch_sampler, num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn, pin_memory=True, prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n        else:\n            data_loader = DataLoader(\n                dataset, batch_size=self.config.batch_size, shuffle=False,\n                num_workers=self.config.num_workers, collate_fn=self._collate_fn,\n                pin_memory=True, prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n        \n        base_gen_config = {\n            \"max_new_tokens\": self.config.max_new_tokens,\n            \"length_penalty\": self.config.length_penalty,\n            \"early_stopping\": self.config.early_stopping,\n            \"use_cache\": True,\n        }\n        if self.config.no_repeat_ngram_size > 0:\n            base_gen_config[\"no_repeat_ngram_size\"] = self.config.no_repeat_ngram_size\n        \n        self.results = []\n        \n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(tqdm(data_loader, desc=\"Translating\")):\n                try:\n                    input_ids = tokenized.input_ids.to(self.config.device)\n                    attention_mask = tokenized.attention_mask.to(self.config.device)\n                    \n                    beam_size = self._get_adaptive_beam_size(input_ids, attention_mask)\n                    gen_config = {**base_gen_config, \"num_beams\": beam_size}\n                    \n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **gen_config)\n                    else:\n                        outputs = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **gen_config)\n                    \n                    translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n                    \n                    if self.config.use_vectorized_postproc:\n                        cleaned = self.postprocessor.postprocess_batch(translations)\n                    else:\n                        cleaned = [self.postprocessor.postprocess_batch([t])[0] for t in translations]\n                    \n                    self.results.extend(zip(batch_ids, cleaned))\n                    \n                    if torch.cuda.is_available() and batch_idx % 10 == 0:\n                        torch.cuda.empty_cache()\n                \n                except Exception as e:\n                    log.error(f\"Batch {batch_idx} error: {e}\")\n                    self.results.extend([(bid, \"\") for bid in batch_ids])\n                    continue\n        \n        log.info(\"Inference completed\")\n        results_df = pd.DataFrame(self.results, columns=[\"id\", \"translation\"])\n        return results_df\n\nlog.info(f\"Loading test data from {cfg.test_data_path}\")\ntest_df = pd.read_csv(cfg.test_data_path, encoding=\"utf-8\")\nlog.info(f\"Loaded {len(test_df)} test samples\")\nprint(f\"\\nTest samples: {len(test_df)}\")\nprint(test_df.head())\n\nengine = UltraInferenceEngine(cfg)\nresults_df = engine.run_inference(test_df)\n\noutput_path = Path(cfg.output_dir) / \"submission.csv\"\nresults_df.to_csv(output_path, index=False)\nprint(f\"\\nSubmission shape: {results_df.shape}\")\nprint(results_df)\nprint(f\"\\nSaved submission.csv\")\nprint(\"Expected score: 35.1 (exact baseline copy)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T14:55:57.936681Z","iopub.execute_input":"2026-02-09T14:55:57.936933Z","iopub.status.idle":"2026-02-09T14:56:48.923155Z","shell.execute_reply.started":"2026-02-09T14:55:57.936912Z","shell.execute_reply":"2026-02-09T14:56:48.922434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}