{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14197093,"sourceType":"datasetVersion","datasetId":9053011},{"sourceId":14217405,"sourceType":"datasetVersion","datasetId":9069004},{"sourceId":14270976,"sourceType":"datasetVersion","datasetId":9107308},{"sourceId":14286298,"sourceType":"datasetVersion","datasetId":9118629},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14385621,"sourceType":"datasetVersion","datasetId":9187326},{"sourceId":14386534,"sourceType":"datasetVersion","datasetId":9187799},{"sourceId":14397071,"sourceType":"datasetVersion","datasetId":9194597},{"sourceId":14398637,"sourceType":"datasetVersion","datasetId":9195725},{"sourceId":14401381,"sourceType":"datasetVersion","datasetId":9197694},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":14418308,"sourceType":"datasetVersion","datasetId":9208781},{"sourceId":14455549,"sourceType":"datasetVersion","datasetId":9233078},{"sourceId":14488088,"sourceType":"datasetVersion","datasetId":9253746},{"sourceId":14654396,"sourceType":"datasetVersion","datasetId":9361597},{"sourceId":14682259,"sourceType":"datasetVersion","datasetId":9353239}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ------------------------------------------------------------\n# STRATEGY 1: Self-Ensemble (Multiple Runs + Voting)\n# ------------------------------------------------------------\n# Idea: Run همین مدل 3 بار با do_sample=True و ensemble کن\n# این می‌تونه variance رو کم کنه و امتیاز رو بالا ببره\n# Expected: 35.0 → 35.1-35.3\n# ------------------------------------------------------------\n\nimport os\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\nos.environ[\"TORCH_CUDNN_V8_API_ENABLED\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\nimport re\nimport logging\nimport warnings\nfrom pathlib import Path\nfrom typing import List\nfrom dataclasses import dataclass\nfrom collections import Counter\n\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings(\"ignore\")\n\nprint(f\"PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}\")\n\n\n@dataclass\nclass SelfEnsembleConfig:\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    model_path: str = \"/kaggle/input/byt5-akkadian-model\"\n    output_dir: str = \"/kaggle/working/\"\n    \n    max_length: int = 512\n    batch_size: int = 8\n    num_workers: int = 4\n    \n    # Proven optimal hyperparameters\n    num_beams: int = 8\n    max_new_tokens: int = 512\n    length_penalty: float = 1.3\n    early_stopping: bool = True\n    \n    # Self-ensemble config\n    num_ensemble_runs: int = 3  # Run 3 times\n    ensemble_strategy: str = \"longest\"  # or \"voting\"\n    \n    use_mixed_precision: bool = True\n    aggressive_postprocessing: bool = True\n    \n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n\n\ndef setup_logging(output_dir: str):\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n    for h in logging.root.handlers[:]:\n        logging.root.removeHandler(h)\n    logging.basicConfig(\n        level=logging.INFO,\n        format=\"%(message)s\",\n        handlers=[logging.StreamHandler()]\n    )\n    return logging.getLogger(__name__)\n\n\nclass OptimizedPreprocessor:\n    def __init__(self):\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|——)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\")\n        }\n    \n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        s = pd.Series(texts).fillna(\"\").astype(str)\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n        return s.tolist()\n\n\nclass VectorizedPostprocessor:\n    def __init__(self, aggressive: bool = True):\n        self.aggressive = aggressive\n        \n        self.patterns = {\n            \"gap\": re.compile(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\.{3,}|…|\\[\\.+\\])\"),\n            \"annotations\": re.compile(r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..\\s*\\w*\\)\", re.I),\n            \"repeated_words\": re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\"),\n            \"whitespace\": re.compile(r\"\\s+\"),\n            \"punct_space\": re.compile(r\"\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\1+\"),\n        }\n        \n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n        self.forbidden_trans = str.maketrans(\"\", \"\", '!?()\"——<>⌈⌋⌊[]+ʾ/;')\n    \n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        s = pd.Series(translations)\n        \n        valid_mask = s.apply(lambda x: isinstance(x, str) and bool(x.strip()))\n        s.loc[~valid_mask] = \"\"\n        \n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n        s = s.str.strip()\n        \n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n            \n            s = s.str.replace(\"<gap>\", \"\\x00G\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\x00B\\x00\", regex=False)\n            s = s.str.translate(self.forbidden_trans)\n            s = s.str.replace(\"\\x00G\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\x00B\\x00\", \" <big_gap> \", regex=False)\n            \n            s = s.str.replace(r\"(\\d+)\\.5\\b\", r\"\\1½\", regex=True)\n            s = s.str.replace(r\"\\b0\\.5\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.25\\b\", r\"\\1¼\", regex=True)\n            s = s.str.replace(r\"\\b0\\.25\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.75\\b\", r\"\\1¾\", regex=True)\n            s = s.str.replace(r\"\\b0\\.75\\b\", \"¾\", regex=True)\n            \n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\1\", regex=True)\n            \n            for n in range(4, 1, -1):\n                pattern = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                s = s.str.replace(pattern, r\"\\1\", regex=True)\n            \n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n        \n        return s.tolist()\n\n\nclass AkkadianDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, preprocessor: OptimizedPreprocessor):\n        self.sample_ids = df[\"id\"].tolist()\n        preprocessed = preprocessor.preprocess_batch(df[\"transliteration\"].tolist())\n        self.input_texts = [\"translate Akkadian to English: \" + text for text in preprocessed]\n    \n    def __len__(self):\n        return len(self.sample_ids)\n    \n    def __getitem__(self, idx: int):\n        return self.sample_ids[idx], self.input_texts[idx]\n\n\nclass SelfEnsembleEngine:\n    def __init__(self, config: SelfEnsembleConfig, logger: logging.Logger):\n        self.config = config\n        self.logger = logger\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(config.aggressive_postprocessing)\n        \n        logger.info(f\"Loading model: {config.model_path}\")\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(config.model_path)\n        self.model = self.model.to(config.device)\n        self.model = self.model.eval()\n        \n        self.tokenizer = AutoTokenizer.from_pretrained(config.model_path)\n        \n        num_params = sum(p.numel() for p in self.model.parameters())\n        logger.info(f\"✅ Model loaded: {num_params:,} parameters\")\n    \n    def _collate_fn(self, batch):\n        ids = [s[0] for s in batch]\n        texts = [s[1] for s in batch]\n        \n        tokenized = self.tokenizer(\n            texts,\n            max_length=self.config.max_length,\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\"\n        )\n        \n        return ids, tokenized\n    \n    def _run_single_inference(self, dataloader) -> dict:\n        \"\"\"Run inference once and return results\"\"\"\n        results = []\n        \n        gen_config = {\n            \"max_new_tokens\": self.config.max_new_tokens,\n            \"length_penalty\": self.config.length_penalty,\n            \"early_stopping\": self.config.early_stopping,\n            \"use_cache\": True,\n            \"num_beams\": self.config.num_beams,\n        }\n        \n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(tqdm(dataloader, desc=\"Inference\", leave=False)):\n                try:\n                    input_ids = tokenized.input_ids.to(self.config.device)\n                    attention_mask = tokenized.attention_mask.to(self.config.device)\n                    \n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(\n                                input_ids=input_ids,\n                                attention_mask=attention_mask,\n                                **gen_config\n                            )\n                    else:\n                        outputs = self.model.generate(\n                            input_ids=input_ids,\n                            attention_mask=attention_mask,\n                            **gen_config\n                        )\n                    \n                    translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n                    cleaned = self.postprocessor.postprocess_batch(translations)\n                    \n                    results.extend(zip(batch_ids, cleaned))\n                    \n                    if torch.cuda.is_available() and batch_idx % 10 == 0:\n                        torch.cuda.empty_cache()\n                \n                except Exception as e:\n                    self.logger.error(f\"❌ Batch {batch_idx} error: {e}\")\n                    results.extend([(bid, \"\") for bid in batch_ids])\n        \n        # Convert to dict for easy lookup\n        return {sample_id: translation for sample_id, translation in results}\n    \n    def _ensemble_results(self, all_results: List[dict]) -> dict:\n        \"\"\"Ensemble multiple inference results\"\"\"\n        sample_ids = list(all_results[0].keys())\n        final_results = {}\n        \n        for sample_id in sample_ids:\n            candidates = [results[sample_id] for results in all_results]\n            \n            if self.config.ensemble_strategy == \"longest\":\n                # Pick the longest translation (more complete)\n                final_results[sample_id] = max(candidates, key=len)\n            \n            elif self.config.ensemble_strategy == \"voting\":\n                # Pick the most common translation\n                counter = Counter(candidates)\n                final_results[sample_id] = counter.most_common(1)[0][0]\n            \n            else:\n                # Default: first one\n                final_results[sample_id] = candidates[0]\n        \n        return final_results\n    \n    def run_inference(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        self.logger.info(f\"🚀 Starting SELF-ENSEMBLE inference ({self.config.num_ensemble_runs} runs)\")\n        \n        dataset = AkkadianDataset(test_df, self.preprocessor)\n        \n        dataloader = DataLoader(\n            dataset,\n            batch_size=self.config.batch_size,\n            shuffle=False,\n            num_workers=self.config.num_workers,\n            collate_fn=self._collate_fn,\n            pin_memory=True,\n            prefetch_factor=2,\n            persistent_workers=self.config.num_workers > 0\n        )\n        \n        # Run inference multiple times\n        all_results = []\n        \n        for run_idx in range(self.config.num_ensemble_runs):\n            self.logger.info(f\"Run {run_idx + 1}/{self.config.num_ensemble_runs}\")\n            results = self._run_single_inference(dataloader)\n            all_results.append(results)\n        \n        # Ensemble the results\n        self.logger.info(f\"Ensembling with strategy: {self.config.ensemble_strategy}\")\n        final_results = self._ensemble_results(all_results)\n        \n        # Convert back to DataFrame\n        results_df = pd.DataFrame([\n            {\"id\": sample_id, \"translation\": translation}\n            for sample_id, translation in final_results.items()\n        ])\n        \n        results_df = self._validate_submission(results_df, test_df[\"id\"].tolist())\n        \n        return results_df\n    \n    def _validate_submission(self, results_df: pd.DataFrame, original_ids: List[int]) -> pd.DataFrame:\n        self.logger.info(\"🔍 Validating...\")\n        \n        results_df[\"id\"] = results_df[\"id\"].astype(int)\n        \n        result_ids = set(results_df[\"id\"].tolist())\n        original_ids_set = set(original_ids)\n        \n        missing_ids = original_ids_set - result_ids\n        if missing_ids:\n            self.logger.warning(f\"⚠️  Missing {len(missing_ids)} IDs\")\n            missing_rows = pd.DataFrame({\n                \"id\": list(missing_ids),\n                \"translation\": [\"<gap>\"] * len(missing_ids)\n            })\n            results_df = pd.concat([results_df, missing_rows], ignore_index=True)\n        \n        results_df = results_df.drop_duplicates(subset=[\"id\"], keep=\"first\")\n        results_df = results_df.sort_values(\"id\").reset_index(drop=True)\n        results_df[\"translation\"] = results_df[\"translation\"].fillna(\"<gap>\")\n        results_df.loc[results_df[\"translation\"].str.strip() == \"\", \"translation\"] = \"<gap>\"\n        \n        self.logger.info(\"✅ Validated\")\n        self._print_report(results_df)\n        \n        return results_df\n    \n    def _print_report(self, df: pd.DataFrame):\n        print(\"\\n\" + \"=\" * 60)\n        print(f\"📊 SELF-ENSEMBLE REPORT ({self.config.num_ensemble_runs} runs)\")\n        print(\"=\" * 60)\n        \n        print(f\"\\nTotal: {len(df)}\")\n        \n        gap_count = (df[\"translation\"] == \"<gap>\").sum()\n        if gap_count > 0:\n            print(f\"Default gaps: {gap_count}\")\n        \n        lengths = df[\"translation\"].astype(str).str.len()\n        print(f\"\\nLength - Mean: {lengths.mean():.1f}, Median: {lengths.median():.1f}\")\n        \n        print(\"\\n📝 Samples:\")\n        for idx in [0, len(df)//2, len(df)-1]:\n            if idx < len(df):\n                row = df.iloc[idx]\n                text = str(row[\"translation\"])\n                preview = text[:70] + \"...\" if len(text) > 70 else text\n                print(f\"  ID {int(row['id']):4d}: {preview}\")\n        \n        print(\"\\n\" + \"=\" * 60 + \"\\n\")\n\n\ndef main():\n    config = SelfEnsembleConfig()\n    logger = setup_logging(config.output_dir)\n    \n    logger.info(\"=\" * 60)\n    logger.info(\"SELF-ENSEMBLE STRATEGY\")\n    logger.info(f\"Runs: {config.num_ensemble_runs}\")\n    logger.info(f\"Strategy: {config.ensemble_strategy}\")\n    logger.info(\"=\" * 60)\n    \n    test_df = pd.read_csv(config.test_data_path)\n    logger.info(f\"✅ Loaded {len(test_df)} test samples\")\n    \n    engine = SelfEnsembleEngine(config, logger)\n    results_df = engine.run_inference(test_df)\n    \n    output_path = Path(config.output_dir) / \"submission.csv\"\n    results_df.to_csv(output_path, index=False)\n    logger.info(f\"✅ Saved to {output_path}\")\n    \n    print(\"\\n🎉 DONE! Self-ensemble with multiple runs\")\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T17:58:07.114102Z","iopub.execute_input":"2026-02-10T17:58:07.114371Z","iopub.status.idle":"2026-02-10T17:58:32.559402Z","shell.execute_reply.started":"2026-02-10T17:58:07.114352Z","shell.execute_reply":"2026-02-10T17:58:32.558682Z"}},"outputs":[],"execution_count":null}]}