{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":14410665,"datasetId":9203761,"databundleVersionId":15226414},{"sourceType":"datasetVersion","sourceId":14376272,"datasetId":9181082,"databundleVersionId":15188111},{"sourceType":"datasetVersion","sourceId":14197093,"datasetId":9053011,"databundleVersionId":14991504},{"sourceType":"datasetVersion","sourceId":14374989,"datasetId":9147887,"databundleVersionId":15186684},{"sourceType":"modelInstanceVersion","sourceId":754941,"databundleVersionId":15740514,"modelInstanceId":573226,"modelId":585575,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":11261,"databundleVersionId":7766209,"modelInstanceId":8332,"modelId":3301,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":734719,"databundleVersionId":15497930,"modelInstanceId":560018,"modelId":572602,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-28T23:48:00.286089Z","iopub.execute_input":"2026-01-28T23:48:00.286311Z","iopub.status.idle":"2026-01-28T23:48:02.516767Z","shell.execute_reply.started":"2026-01-28T23:48:00.286287Z","shell.execute_reply":"2026-01-28T23:48:02.515996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===========================\n# IMPORTS ESSENCIAIS\n# ===========================\nimport os\nimport re\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\nfrom functools import lru_cache\nfrom pathlib import Path\n\n# ===========================\n# CONFIGURAÇÃO\n# ===========================\nclass Config:\n    def __init__(self):\n        # Caminhos dos modelos (já carregados no Kaggle)\n        self.model_paths = [\n            \"/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr/pytorch/default/6\",\n            \"/kaggle/input/byt5-base-big-data2\", \n            \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\",\n            # \"/kaggle/input/oracc-akkadian-english-parallel-corpus\",\n        ]\n        \n        # Pesos para ensemble (model soup) - ajuste conforme necessário\n        self.model_weights = [0.5, 0.4, 0.1] #[0.375, 0.3125, 0.3125]\n        \n        # Parâmetros de geração\n        self.generation_config = {\n            \"num_beams\": 8,\n            \"max_new_tokens\": 512,\n            \"min_length\": 20,\n            \"length_penalty\": 1.096,\n            \"repetition_penalty\": 1.0,\n            \"early_stopping\": True,\n            \"no_repeat_ngram_size\": 3,\n            \"temperature\": 0.1,\n            \"do_sample\": False,\n        }\n        \n        # Configurações técnicas\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        self.max_len = 512\n        self.batch_size = 8 if torch.cuda.is_available() else 4\n        self.data_path = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n        \n        # Validação\n        self.min_translation_length = 5\n        self.max_translation_length = 500\n\n# ===========================\n# PROCESSADOR DE TEXTO\n# ===========================\nclass TextProcessor:\n    \"\"\"Pré e pós-processamento especializado para texto acadiano\"\"\"\n    \n    # Padrões compilados para performance\n    BIG_GAP_PATTERN = re.compile(r\"(\\.{3,}|…+|……)\")\n    SMALL_GAP_PATTERN = re.compile(r\"(xx+|\\s+x\\s+)\")\n    PAREN_NOTES = re.compile(r\"\\((?:fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)\", re.I)\n    REPEATED_WORD = re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\")\n    FRACTIONS = {\n        r\"\\.5\\b\": \" ½\",\n        r\"\\.25\\b\": \" ¼\", \n        r\"\\.75\\b\": \" ¾\",\n        r\"\\.33+\\d*\\b\": \" ⅓\",\n        r\"\\.66+\\d*\\b\": \" ⅔\",\n    }\n    \n    @staticmethod\n    def preprocess_transliteration(text: str) -> str:\n        \"\"\"Pré-processa texto acadiano\"\"\"\n        if pd.isna(text):\n            return \"\"\n        \n        text = str(text)\n        \n        # Normalizar lacunas\n        text = TextProcessor.BIG_GAP_PATTERN.sub(\"<big_gap>\", text)\n        text = TextProcessor.SMALL_GAP_PATTERN.sub(\"<gap>\", text)\n        \n        # Normalizar espaços\n        text = ' '.join(text.split())\n        \n        return text\n    \n    @staticmethod\n    def postprocess_translation(text: str, aggressive: bool = False) -> str:\n        \"\"\"Pós-processa tradução\"\"\"\n        if not isinstance(text, str) or not text.strip():\n            return \"\"\n        \n        text = text.strip()\n        \n        # Normalização de caracteres\n        text = text.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n        sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        text = text.translate(sub_map)\n        \n        # Normalização de lacunas\n        text = re.sub(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", \"<gap>\", text, flags=re.I)\n        text = re.sub(r\"(\\.{3,}|…|\\[\\.+\\])\", \"<big_gap>\", text)\n        \n        # Limpeza condicional\n        if aggressive:\n            text = TextProcessor.PAREN_NOTES.sub(\"\", text)\n        \n        # Proteger tokens especiais\n        text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n        text = text.replace(\"<big_gap>\", \"\\x00BIG\\x00\")\n        \n        # Limpeza de caracteres problemáticos\n        bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n        text = text.translate(str.maketrans(\"\", \"\", bad_chars))\n        \n        # Restaurar tokens\n        text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n        text = text.replace(\"\\x00BIG\\x00\", \" <big_gap> \")\n        \n        # Converter frações decimais\n        for pattern, replacement in TextProcessor.FRACTIONS.items():\n            text = re.sub(r\"(\\d+)\" + pattern, r\"\\1\" + replacement, text)\n            text = re.sub(r\"\\b0\" + pattern, replacement.strip(), text)\n        \n        # Remover repetições (se agressivo)\n        if aggressive:\n            text = TextProcessor.REPEATED_WORD.sub(r\"\\1\", text)\n            for n in range(4, 1, -1):\n                pattern = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                text = re.sub(pattern, r\"\\1\", text)\n        \n        # Normalização final\n        text = re.sub(r\"\\s+\", \" \", text).strip()\n        \n        return text\n    \n    @staticmethod\n    def validate_translation(text: str, min_len: int = 5, max_len: int = 500) -> bool:\n        \"\"\"Valida se a tradução parece válida\"\"\"\n        if not text or len(text) < min_len or len(text) > max_len:\n            return False\n        \n        # Verificar proporção de tokens especiais\n        special_tokens = len(re.findall(r\"<gap>|<big_gap>\", text))\n        if special_tokens > len(text.split()) * 0.5:\n            return False\n            \n        # Verificar caracteres inválidos\n        invalid_chars = re.findall(r\"[^\\w\\s.,;:!?<>-]\", text)\n        if len(invalid_chars) > len(text) * 0.1:\n            return False\n            \n        return True\n\n# ===========================\n# GESTOR DE MODELOS\n# ===========================\nclass ModelManager:\n    \"\"\"Gerencia carregamento de modelos locais\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n    \n    @lru_cache(maxsize=4)\n    def load_model(self, model_path: str):\n        \"\"\"Carrega modelo com cache em memória\"\"\"\n        print(f\"📥 Carregando: {Path(model_path).name}\")\n        model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n        model = model.to(self.config.device)\n        model.eval()\n        return model\n    \n    def create_ensemble_soup(self, model_paths: list, weights: list):\n        \"\"\"Cria ensemble via model soup - VERSÃO FINAL CORRIGIDA\"\"\"\n        # 1. Verificação básica de entrada\n        if len(model_paths) != len(weights):\n            raise ValueError(f\"Número de modelos ({len(model_paths)}) e pesos ({len(weights)}) não correspondem.\")\n    \n        # 2. Normalizar pesos\n        weights = np.array(weights) / sum(weights)\n        print(f\"🔢 Pesos normalizados: {weights}\")\n    \n        # 3. Carregar todos os modelos\n        models = []\n        for path in tqdm(model_paths, desc=\"📥 Carregando modelos\"):\n            model = self.load_model(path)\n            models.append(model)\n    \n        print(f\"✅ Modelos carregados: {len(models)}\")\n    \n        # 4. Caso especial: apenas um modelo\n        if len(models) == 1:\n            print(\"ℹ️  Apenas um modelo, retornando sem combinação.\")\n            return models[0]\n    \n        # 5. Usar primeiro modelo como base para o soup\n        base_model = models[0]\n        soup_state_dict = base_model.state_dict()\n    \n        print(\"🧮 Combinando pesos dos modelos...\")\n    \n        # 6. CORREÇÃO PRINCIPAL: Inicializar com zeros e depois somar contribuições\n        # Primeiro, zerar todos os tensores no state_dict base\n        for key in soup_state_dict:\n            soup_state_dict[key] = torch.zeros_like(soup_state_dict[key])\n    \n        # 7. Somar contribuições ponderadas de CADA modelo\n        for i, model in enumerate(models):\n            state_dict = model.state_dict()\n            model_weight = weights[i]\n        \n            for key in soup_state_dict:\n                if key in state_dict and state_dict[key].shape == soup_state_dict[key].shape:\n                    soup_state_dict[key] += model_weight * state_dict[key]\n                elif key in state_dict:\n                    print(f\"⚠️  Shape incompatível para {key} no modelo {i}. Ignorando.\")\n    \n        # 8. Carregar os pesos combinados de volta\n        base_model.load_state_dict(soup_state_dict)\n    \n        # 9. CORREÇÃO DO ERRO: Liberar memória SEM deletar por índice\n        # Método seguro: deletar referências e esvaziar a lista\n        print(\"🧹 Liberando memória dos modelos auxiliares...\")\n    \n        # Primeiro, manter apenas uma referência ao base_model\n        models_to_delete = models[1:]  # Todos exceto o primeiro (que é o base_model)\n    \n        # Deletar cada modelo explicitamente\n        for model in models_to_delete:\n            del model\n    \n        # Esvaziar a lista\n        models.clear()\n    \n        # Coletar lixo\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n    \n        print(\"✅ Ensemble criado com sucesso!\")\n        return base_model\n\n# ===========================\n# DATASET E DATALOADER\n# ===========================\nclass AkkadianDataset(Dataset):\n    \"\"\"Dataset para tradução acadiana\"\"\"\n    \n    def __init__(self, dataframe, tokenizer, max_length=512):\n        self.ids = dataframe[\"id\"].tolist()\n        self.texts = dataframe[\"transliteration\"].tolist()\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        \n        # Pré-tokenizar para performance\n        self.encoded_texts = self._pre_tokenize()\n    \n    def _pre_tokenize(self):\n        \"\"\"Pré-tokeniza textos\"\"\"\n        encoded = self.tokenizer(\n            [f\"translate Akkadian to English: {t}\" for t in self.texts],\n            max_length=self.max_length,\n            padding=False,\n            truncation=True,\n            return_tensors=None,\n        )\n        return encoded[\"input_ids\"]\n    \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        return {\n            \"id\": self.ids[idx],\n            \"input_ids\": self.encoded_texts[idx],\n            \"text\": self.texts[idx]\n        }\n\ndef dynamic_collate_fn(batch, tokenizer):\n    \"\"\"Função de collate dinâmica\"\"\"\n    ids = [item[\"id\"] for item in batch]\n    input_ids = [item[\"input_ids\"] for item in batch]\n    \n    # Padding dinâmico\n    padded = tokenizer.pad(\n        {\"input_ids\": input_ids},\n        padding=True,\n        return_tensors=\"pt\"\n    )\n    \n    return ids, padded\n\n# ===========================\n# MOTOR DE INFERÊNCIA\n# ===========================\nclass InferenceEngine:\n    \"\"\"Motor de inferência com validação\"\"\"\n    \n    def __init__(self, model, tokenizer, config):\n        self.model = model\n        self.tokenizer = tokenizer\n        self.config = config\n        self.processor = TextProcessor()\n        \n    def batch_inference(self, data_loader, progress_bar=True):\n        \"\"\"Inferência em batch com validação\"\"\"\n        results = []\n        self.model.eval()\n        \n        iterator = tqdm(data_loader, desc=\"Inferência\") if progress_bar else data_loader\n        \n        with torch.no_grad(), torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n            for batch_ids, batch_inputs in iterator:\n                # Mover para dispositivo\n                input_ids = batch_inputs[\"input_ids\"].to(self.config.device)\n                attention_mask = batch_inputs[\"attention_mask\"].to(self.config.device)\n                \n                # Gerar traduções\n                outputs = self.model.generate(\n                    input_ids=input_ids,\n                    attention_mask=attention_mask,\n                    **self.config.generation_config\n                )\n                \n                # Decodificar\n                decoded = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n                \n                # Pós-processar e validar\n                for id_, text in zip(batch_ids, decoded):\n                    processed = self.processor.postprocess_translation(text, aggressive=False)\n                    \n                    # Validar e corrigir se necessário\n                    if not self.processor.validate_translation(processed):\n                        processed = self.processor.postprocess_translation(text, aggressive=True)\n                    \n                    results.append((id_, processed))\n        \n        return results\n\n# ===========================\n# PIPELINE PRINCIPAL\n# ===========================\nclass AkkadianTranslationPipeline:\n    \"\"\"Pipeline completo de tradução acadiana\"\"\"\n    \n    def __init__(self, config=None):\n        self.config = config or Config()\n        self.model_manager = ModelManager(self.config)\n        self.processor = TextProcessor()\n        \n    def load_data(self, data_path):\n        \"\"\"Carrega e pré-processa dados\"\"\"\n        df = pd.read_csv(data_path)\n        \n        # Aplicar pré-processamento\n        df[\"transliteration\"] = df[\"transliteration\"].apply(\n            self.processor.preprocess_transliteration\n        )\n        \n        print(f\"📊 Total de amostras: {len(df)}\")\n        print(f\"📊 Comprimento médio: {df['transliteration'].str.len().mean():.1f} caracteres\")\n        \n        return df\n    \n    def prepare_model(self):\n        \"\"\"Prepara modelo com ensemble soup\"\"\"\n        print(\"🧠 Preparando ensemble (model soup)...\")\n        model = self.model_manager.create_ensemble_soup(\n            self.config.model_paths,\n            self.config.model_weights\n        )\n        tokenizer = AutoTokenizer.from_pretrained(self.config.model_paths[0])\n        return model, tokenizer\n    \n    def create_dataloader(self, df, tokenizer):\n        \"\"\"Cria dataloader otimizado\"\"\"\n        dataset = AkkadianDataset(df, tokenizer, self.config.max_len)\n        data_loader = DataLoader(\n            dataset,\n            batch_size=self.config.batch_size,\n            shuffle=False,\n            collate_fn=lambda b: dynamic_collate_fn(b, tokenizer),\n            num_workers=2,\n            pin_memory=True\n        )\n        return data_loader\n    \n    def create_submission(self, predictions, output_path=\"submission.csv\"):\n        \"\"\"Cria arquivo de submissão com formatação estrita para a competição\"\"\"\n        # Converter previsões para DataFrame\n        df = pd.DataFrame(predictions, columns=[\"id\", \"translation\"])\n    \n        # 1. Garantir que a coluna 'id' seja do tipo inteiro\n        df[\"id\"] = pd.to_numeric(df[\"id\"], errors='coerce').astype('int64')\n    \n        # 2. Ordenar estritamente por 'id' (ordem crescente)\n        df = df.sort_values(\"id\").reset_index(drop=True)\n    \n        # 3. Garantir que NENHUMA tradução esteja vazia\n        #    Substitui NaN, strings vazias ou apenas espaços por um placeholder\n        placeholder = \"Translation pending review.\"\n        df[\"translation\"] = df[\"translation\"].fillna(placeholder)\n        df.loc[df[\"translation\"].str.strip() == \"\", \"translation\"] = placeholder\n    \n        # 4. Garantir que cada tradução seja uma única string de linha\n        #    Remove quebras de linha e espaços múltiplos internos\n        df[\"translation\"] = df[\"translation\"].apply(\n            lambda x: ' '.join(str(x).replace('\\n', ' ').split())\n        )\n    \n        # 5. Salvar CSV com formato exato: duas colunas, cabeçalho, sem índice\n        df.to_csv(output_path, index=False)\n    \n        # 6. Validação final e log\n        original_test_df = pd.read_csv(self.config.data_path)\n    \n        print(f\"✅ Arquivo salvo: {output_path}\")\n        print(f\"📊 Linhas no teste original: {len(original_test_df)}\")\n        print(f\"📊 Linhas na submissão gerada: {len(df)}\")\n        print(f\"📊 IDs no teste: {original_test_df['id'].min()} a {original_test_df['id'].max()}\")\n        print(f\"📊 IDs na submissão: {df['id'].min()} a {df['id'].max()}\")\n        print(f\"🔍 Traduções vazias corrigidas: {(df['translation'] == placeholder).sum()}\")\n    \n        # Verificação crítica: o número de linhas DEVE ser idêntico\n        if len(df) != len(original_test_df):\n            print(f\"❌ ERRO CRÍTICO: A submissão tem {len(df)} linhas, mas o arquivo de teste tem {len(original_test_df)}.\")\n            print(\"   Ajustando para incluir todos os IDs do arquivo de teste...\")\n        \n            # Estratégia de fallback: criar um DataFrame com todos os IDs do teste\n            # e fazer um merge left para garantir que nenhum ID esteja faltando.\n            all_ids_df = original_test_df[[\"id\"]].copy()\n            df_corrected = all_ids_df.merge(df, on=\"id\", how=\"left\")\n            df_corrected[\"translation\"] = df_corrected[\"translation\"].fillna(placeholder)\n            df_corrected.to_csv(output_path, index=False)\n            print(f\"✅ Submissão regenerada com {len(df_corrected)} linhas.\")\n            df = df_corrected\n    \n        # Amostra das primeiras linhas para conferência\n        print(\"\\n🔍 Amostra do arquivo gerado (primeiras 3 linhas):\")\n        print(df.head(3).to_string(index=False))\n    \n        return df\n    \n    def run(self):\n        \"\"\"Executa pipeline completo\"\"\"\n        print(\"=\" * 60)\n        print(\"🚀 INICIANDO PIPELINE DE TRADUÇÃO ACADIANA\")\n        print(\"=\" * 60)\n        \n        # 1. Carregar dados\n        print(\"\\n[1/4] 📂 Carregando dados...\")\n        df = self.load_data(self.config.data_path)\n        \n        # 2. Preparar modelo\n        print(\"\\n[2/4] 🧠 Preparando modelo...\")\n        model, tokenizer = self.prepare_model()\n        \n        # 3. Criar dataloader\n        print(\"\\n[3/4] ⚙️ Preparando dataloader...\")\n        data_loader = self.create_dataloader(df, tokenizer)\n        \n        # 4. Executar inferência\n        print(\"\\n[4/4] 🔮 Executando inferência...\")\n        inference_engine = InferenceEngine(model, tokenizer, self.config)\n        predictions = inference_engine.batch_inference(data_loader)\n        \n        # 5. Criar submissão\n        submission = self.create_submission(predictions)\n        \n        print(\"\\n\" + \"=\" * 60)\n        print(\"🎉 PIPELINE CONCLUÍDO COM SUCESSO!\")\n        print(\"=\" * 60)\n        \n        return submission\n\n# ===========================\n# EXECUÇÃO PRINCIPAL\n# ===========================\ndef main():\n    \"\"\"Função principal\"\"\"\n    \n    # Inicializar pipeline\n    pipeline = AkkadianTranslationPipeline()\n    \n    # Executar pipeline\n    try:\n        submission = pipeline.run()\n        \n        # Limpeza de memória\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        gc.collect()\n            \n    except Exception as e:\n        print(f\"❌ Erro durante execução: {e}\")\n        raise\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T23:48:02.518675Z","iopub.execute_input":"2026-01-28T23:48:02.519332Z","iopub.status.idle":"2026-01-28T23:49:57.367633Z","shell.execute_reply.started":"2026-01-28T23:48:02.519288Z","shell.execute_reply":"2026-01-28T23:49:57.360641Z"}},"outputs":[],"execution_count":null}]}