{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14197093,"sourceType":"datasetVersion","datasetId":9053011},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":11261,"sourceType":"modelInstanceVersion","modelInstanceId":8332,"modelId":3301},{"sourceId":734719,"sourceType":"modelInstanceVersion","modelInstanceId":560018,"modelId":572602}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-31T14:20:01.425013Z","iopub.execute_input":"2026-01-31T14:20:01.425310Z","iopub.status.idle":"2026-01-31T14:20:01.746877Z","shell.execute_reply.started":"2026-01-31T14:20:01.425282Z","shell.execute_reply":"2026-01-31T14:20:01.746126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===========================\n# IMPORTS ESSENCIAIS\n# ===========================\nimport os\nimport sys\nimport re\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\nfrom functools import lru_cache\nfrom pathlib import Path\n\n# ===========================\n# CONFIGURAÇÃO ROBUSTA\n# ===========================\nclass Config:\n    def __init__(self):\n        # Caminhos dos modelos com fallback\n        self.model_paths = []\n        \n        # Verificar quais modelos estão disponíveis\n        possible_paths = [\n            \"/kaggle/input/byt5-akkadian-model\",\n            \"/kaggle/input/byt5-base-big-data2\", \n            \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\",\n        ]\n        \n        for path in possible_paths:\n            if os.path.exists(path):\n                self.model_paths.append(path)\n                print(f\"✅ Modelo encontrado: {path}\")\n        \n        if not self.model_paths:\n            raise ValueError(\"❌ Nenhum modelo encontrado!\")\n        \n        print(f\"📊 Modelos disponíveis: {len(self.model_paths)}\")\n        \n        # Pesos iguais se não tivermos 3 modelos\n        if len(self.model_paths) == 3:\n            self.model_weights = [0.9, 0.08, 0.02] #[0.375, 0.3125, 0.3125]\n        elif len(self.model_paths) == 2:\n            self.model_weights = [0.9, 0.1] #[0.5, 0.5]\n        else:\n            self.model_weights = [1.0]\n        \n        # Parâmetros de geração mais conservadores\n        self.generation_config = {\n            \"num_beams\": 4,  # Reduzido para economizar memória\n            \"max_new_tokens\": 256,\n            \"min_length\": 10,\n            \"length_penalty\": 1.0,\n            \"repetition_penalty\": 1.1,\n            \"early_stopping\": True,\n            \"no_repeat_ngram_size\": 3,\n            \"temperature\": 0.8,\n            \"do_sample\": False,\n        }\n        \n        # Configurações técnicas\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        print(f\"🖥️  Dispositivo: {self.device}\")\n        \n        # Verificar memória GPU\n        if torch.cuda.is_available():\n            gpu_mem = torch.cuda.get_device_properties(0).total_memory / 1e9\n            print(f\"🧠 Memória GPU: {gpu_mem:.1f} GB\")\n            if gpu_mem < 16:  # GPU com menos memória\n                self.batch_size = 4\n                self.max_len = 256\n                print(f\"⚠️  GPU com memória limitada. Ajustando batch_size para {self.batch_size}\")\n            else:\n                self.batch_size = 8\n                self.max_len = 512\n        else:\n            self.batch_size = 2\n            self.max_len = 256\n        \n        # Encontrar arquivo de teste\n        self.data_path = self.find_test_file()\n        \n        # Validação\n        self.min_translation_length = 3  # Reduzido para aceitar traduções curtas\n        self.max_translation_length = 300\n    \n    def find_test_file(self):\n        \"\"\"Encontra o arquivo de teste automaticamente\"\"\"\n        possible_paths = [\n            \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n            \"/kaggle/input/test/test.csv\",\n            \"/kaggle/input/data/test.csv\",\n            \"/kaggle/input/competition-data/test.csv\",\n        ]\n        \n        for path in possible_paths:\n            if os.path.exists(path):\n                print(f\"📁 Arquivo de teste encontrado: {path}\")\n                return path\n        \n        # Tentar encontrar recursivamente\n        for dirpath, dirnames, filenames in os.walk(\"/kaggle/input\"):\n            if \"test.csv\" in filenames:\n                found_path = os.path.join(dirpath, \"test.csv\")\n                print(f\"📁 Arquivo de teste encontrado (busca recursiva): {found_path}\")\n                return found_path\n        \n        raise FileNotFoundError(\"❌ Arquivo test.csv não encontrado!\")\n\n# ===========================\n# PROCESSADOR DE TEXTO\n# ===========================\nclass TextProcessor:\n    \"\"\"Pré e pós-processamento especializado para texto acadiano\"\"\"\n    \n    # Padrões compilados para performance\n    BIG_GAP_PATTERN = re.compile(r\"(\\.{3,}|\\u2026+|\\u2026\\u2026)\")\n    SMALL_GAP_PATTERN = re.compile(r\"(xx+|\\s+x\\s+)\")\n    PAREN_NOTES = re.compile(r\"\\((?:fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)\", re.I)\n    REPEATED_WORD = re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\")\n    \n    @staticmethod\n    def preprocess_transliteration(text: str) -> str:\n        \"\"\"Pré-processa texto acadiano\"\"\"\n        if pd.isna(text):\n            return \"\"\n        \n        text = str(text)\n        \n        # Normalizar lacunas\n        text = TextProcessor.BIG_GAP_PATTERN.sub(\"<big_gap>\", text)\n        text = TextProcessor.SMALL_GAP_PATTERN.sub(\"<gap>\", text)\n        \n        # Normalizar espaços\n        text = ' '.join(text.split())\n        \n        return text\n    \n    @staticmethod\n    def postprocess_translation(text: str, aggressive: bool = False) -> str:\n        \"\"\"Pós-processa tradução\"\"\"\n        if not isinstance(text, str) or not text.strip():\n            return \"\"\n        \n        text = text.strip()\n        \n        # Normalização básica\n        text = text.replace(\"\\u1e2b\", \"h\").replace(\"\\u1e2a\", \"H\")\n        \n        # Normalizar lacunas\n        text = re.sub(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", \"<gap>\", text, flags=re.I)\n        text = re.sub(r\"(\\.{3,}|\\u2026|\\[\\.+\\])\", \"<big_gap>\", text)\n        \n        # Proteger tokens especiais\n        text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n        text = text.replace(\"<big_gap>\", \"\\x00BIG\\x00\")\n        \n        # Limpeza de caracteres problemáticos (mais conservadora)\n        bad_chars = '!?()\"<>[]+/;'\n        text = text.translate(str.maketrans(\"\", \"\", bad_chars))\n        \n        # Restaurar tokens\n        text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n        text = text.replace(\"\\x00BIG\\x00\", \" <big_gap> \")\n        \n        # Normalização final\n        text = re.sub(r\"\\s+\", \" \", text).strip()\n        \n        return text\n    \n    @staticmethod\n    def validate_translation(text: str, min_len: int = 3, max_len: int = 300) -> bool:\n        \"\"\"Valida se a tradução parece válida\"\"\"\n        if not text or len(text) < min_len or len(text) > max_len:\n            return False\n        \n        # Verificar proporção de tokens especiais\n        special_tokens = len(re.findall(r\"<gap>|<big_gap>\", text))\n        if special_tokens > len(text.split()) * 0.5:\n            return False\n            \n        return True\n\n# ===========================\n# GESTOR DE MODELOS\n# ===========================\nclass ModelManager:\n    \"\"\"Gerencia carregamento de modelos locais\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n    \n    @lru_cache(maxsize=4)\n    def load_model(self, model_path: str):\n        \"\"\"Carrega modelo com cache em memória\"\"\"\n        try:\n            print(f\"📦 Carregando: {Path(model_path).name}\")\n            model = AutoModelForSeq2SeqLM.from_pretrained(\n                model_path,\n                torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32\n            )\n            model = model.to(self.config.device)\n            model.eval()\n            return model\n        except Exception as e:\n            print(f\"⚠️  Erro ao carregar modelo {model_path}: {e}\")\n            return None\n    \n    def create_ensemble_soup(self, model_paths: list, weights: list):\n        \"\"\"Cria ensemble via model soup - versão robusta\"\"\"\n        if len(model_paths) != len(weights):\n            weights = [1.0/len(model_paths)] * len(model_paths)\n        \n        # Carregar modelos disponíveis\n        models = []\n        valid_paths = []\n        valid_weights = []\n        \n        for i, path in enumerate(model_paths):\n            model = self.load_model(path)\n            if model is not None:\n                models.append(model)\n                valid_paths.append(path)\n                valid_weights.append(weights[i])\n        \n        if not models:\n            raise ValueError(\"❌ Nenhum modelo carregado com sucesso!\")\n        \n        print(f\"✅ Modelos carregados: {len(models)}\")\n        \n        # Normalizar pesos\n        valid_weights = np.array(valid_weights) / sum(valid_weights)\n        print(f\"⚖️  Pesos normalizados: {valid_weights}\")\n        \n        # Caso especial: apenas um modelo\n        if len(models) == 1:\n            print(\"ℹ️  Apenas um modelo, retornando sem combinação.\")\n            return models[0]\n        \n        # Usar primeiro modelo como base\n        base_model = models[0]\n        soup_state_dict = base_model.state_dict()\n        \n        print(\"🧠 Combinando pesos dos modelos...\")\n        \n        # Inicializar com zeros\n        for key in soup_state_dict:\n            soup_state_dict[key] = torch.zeros_like(soup_state_dict[key])\n        \n        # Somar contribuições ponderadas\n        for i, model in enumerate(models):\n            state_dict = model.state_dict()\n            model_weight = valid_weights[i]\n        \n            for key in soup_state_dict:\n                if key in state_dict and state_dict[key].shape == soup_state_dict[key].shape:\n                    soup_state_dict[key] += model_weight * state_dict[key]\n        \n        # Carregar pesos combinados\n        base_model.load_state_dict(soup_state_dict)\n        \n        # Limpar memória\n        print(\"🧹 Liberando memória...\")\n        for model in models[1:]:\n            del model\n        \n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        \n        print(\"✅ Ensemble criado com sucesso!\")\n        return base_model\n\n# ===========================\n# PIPELINE PRINCIPAL ROBUSTA\n# ===========================\nclass RobustAkkadianTranslationPipeline:\n    \"\"\"Pipeline robusta de tradução acadiana\"\"\"\n    \n    def __init__(self, config=None):\n        self.config = config or Config()\n        self.model_manager = ModelManager(self.config)\n        self.processor = TextProcessor()\n        \n    def load_data(self):\n        \"\"\"Carrega e pré-processa dados com validação\"\"\"\n        try:\n            df = pd.read_csv(self.config.data_path)\n            \n            # Verificar colunas necessárias\n            if \"id\" not in df.columns or \"transliteration\" not in df.columns:\n                print(\"⚠️  Colunas esperadas não encontradas. Verificando alternativas...\")\n                \n                # Tentar encontrar colunas por padrão\n                for col in df.columns:\n                    if \"id\" in col.lower():\n                        df = df.rename(columns={col: \"id\"})\n                    elif \"transliteration\" in col.lower() or \"text\" in col.lower():\n                        df = df.rename(columns={col: \"transliteration\"})\n            \n            # Garantir que temos as colunas necessárias\n            if \"id\" not in df.columns:\n                df[\"id\"] = range(len(df))\n            \n            if \"transliteration\" not in df.columns:\n                raise ValueError(\"❌ Coluna 'transliteration' não encontrada no arquivo de teste\")\n            \n            # Pré-processamento\n            df[\"transliteration\"] = df[\"transliteration\"].apply(\n                self.processor.preprocess_transliteration\n            )\n            \n            print(f\"📊 Total de amostras: {len(df)}\")\n            print(f\"📏 Comprimento médio: {df['transliteration'].str.len().mean():.1f} caracteres\")\n            \n            return df\n            \n        except Exception as e:\n            print(f\"❌ Erro ao carregar dados: {e}\")\n            raise\n    \n    def run(self):\n        \"\"\"Executa pipeline completa com tratamento de erros\"\"\"\n        print(\"=\" * 60)\n        print(\"🚀 INICIANDO PIPELINE DE TRADUÇÃO ACADIANA\")\n        print(\"=\" * 60)\n        \n        try:\n            # 1. Carregar dados\n            print(\"\\n[1/4] 📂 Carregando dados...\")\n            df = self.load_data()\n            \n            # 2. Preparar modelo e tokenizer\n            print(\"\\n[2/4] 🤖 Preparando modelo...\")\n            model = self.model_manager.create_ensemble_soup(\n                self.config.model_paths,\n                self.config.model_weights\n            )\n            \n            # Carregar tokenizer do primeiro modelo\n            tokenizer_path = self.config.model_paths[0]\n            tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n            \n            # 3. Inferência\n            print(\"\\n[3/4] ⚡ Executando inferência...\")\n            predictions = []\n            \n            model.eval()\n            with torch.no_grad():\n                for idx, row in tqdm(df.iterrows(), total=len(df), desc=\"Traduzindo\"):\n                    try:\n                        # Preparar entrada\n                        input_text = f\"translate Akkadian to English: {row['transliteration']}\"\n                        inputs = tokenizer(\n                            input_text,\n                            max_length=self.config.max_len,\n                            truncation=True,\n                            padding=True,\n                            return_tensors=\"pt\"\n                        ).to(self.config.device)\n                        \n                        # Gerar tradução\n                        outputs = model.generate(\n                            **inputs,\n                            **self.config.generation_config\n                        )\n                        \n                        # Decodificar\n                        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n                        \n                        # Pós-processar\n                        translation = self.processor.postprocess_translation(translation)\n                        \n                        # Validar\n                        if not self.processor.validate_translation(translation):\n                            translation = \"Translation requires review.\"\n                        \n                        predictions.append((row[\"id\"], translation))\n                        \n                    except Exception as e:\n                        print(f\"⚠️  Erro na amostra {row['id']}: {e}\")\n                        predictions.append((row[\"id\"], \"Translation pending.\"))\n            \n            # 4. Criar submissão\n            print(\"\\n[4/4] 💾 Criando arquivo de submissão...\")\n            submission_df = self.create_submission(predictions)\n            \n            print(\"\\n\" + \"=\" * 60)\n            print(\"🎉 PIPELINE CONCLUÍDA COM SUCESSO!\")\n            print(\"=\" * 60)\n            \n            return submission_df\n            \n        except Exception as e:\n            print(f\"\\n❌ ERRO CRÍTICO: {e}\")\n            print(\"Tentando criar submissão de fallback...\")\n            \n            # Criar submissão de fallback\n            fallback_predictions = [(i, \"Translation pending review.\") for i in range(1000)]\n            return self.create_submission(fallback_predictions)\n    \n    def create_submission(self, predictions, output_path=\"submission.csv\"):\n        \"\"\"Cria arquivo de submissão robusto\"\"\"\n        try:\n            # Converter para DataFrame\n            df = pd.DataFrame(predictions, columns=[\"id\", \"translation\"])\n            \n            # Garantir tipos corretos\n            df[\"id\"] = pd.to_numeric(df[\"id\"], errors=\"coerce\").fillna(0).astype(int)\n            \n            # Ordenar por ID\n            df = df.sort_values(\"id\").reset_index(drop=True)\n            \n            # Preencher valores vazios\n            placeholder = \"Translation pending review.\"\n            df[\"translation\"] = df[\"translation\"].fillna(placeholder)\n            df.loc[df[\"translation\"].str.strip() == \"\", \"translation\"] = placeholder\n            \n            # Limpar formatação\n            df[\"translation\"] = df[\"translation\"].apply(\n                lambda x: \" \".join(str(x).replace(\"\\n\", \" \").split())\n            )\n            \n            # Salvar\n            df.to_csv(output_path, index=False)\n            \n            # Log\n            print(f\"✅ Arquivo salvo: {output_path}\")\n            print(f\"📈 Linhas na submissão: {len(df)}\")\n            print(f\"🔍 IDs: {df['id'].min()} a {df['id'].max()}\")\n            \n            # Amostra\n            print(\"\\n📋 Amostra (primeiras 3 linhas):\")\n            print(df.head(3).to_string(index=False))\n            \n            return df\n            \n        except Exception as e:\n            print(f\"❌ Erro ao criar submissão: {e}\")\n            \n            # Criar submissão mínima de emergência\n            emergency_df = pd.DataFrame({\n                \"id\": list(range(1000)),\n                \"translation\": [\"Translation pending review.\"] * 1000\n            })\n            emergency_df.to_csv(output_path, index=False)\n            print(f\"⚠️  Submissão de emergência criada: {output_path}\")\n            \n            return emergency_df\n\n# ===========================\n# EXECUÇÃO PRINCIPAL\n# ===========================\ndef main():\n    \"\"\"Função principal com tratamento robusto de erros\"\"\"\n    print(\"Iniciando tradução acadiana...\")\n    \n    try:\n        # Inicializar pipeline\n        pipeline = RobustAkkadianTranslationPipeline()\n        \n        # Executar\n        submission = pipeline.run()\n        \n        # Limpeza final\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        \n        print(\"\\n✅ Processo finalizado!\")\n        return submission\n        \n    except Exception as e:\n        print(f\"\\n❌ ERRO NO PROCESSO PRINCIPAL: {e}\")\n        print(\"Criando submissão de fallback...\")\n        \n        # Fallback final\n        fallback_df = pd.DataFrame({\n            \"id\": list(range(1000)),\n            \"translation\": [\"Translation pending review.\"] * 1000\n        })\n        fallback_df.to_csv(\"submission.csv\", index=False)\n        return fallback_df\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T14:20:01.748494Z","iopub.execute_input":"2026-01-31T14:20:01.748901Z","iopub.status.idle":"2026-01-31T14:20:04.032033Z","shell.execute_reply.started":"2026-01-31T14:20:01.748877Z","shell.execute_reply":"2026-01-31T14:20:04.031172Z"}},"outputs":[],"execution_count":null}]}