{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15340418,"datasetId":9811321,"databundleVersionId":16249037},{"sourceType":"datasetVersion","sourceId":15319436,"datasetId":9362980,"databundleVersionId":16225436}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # ============================================================================\n# # 🔍 简单推理测试\n# # ============================================================================\n\n# from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n# # 加载模型\n# print(\"📥 加载 EMA 模型...\")\n# model_path = \"/kaggle/input/notebooks/wukeneth/train-v6/output_byt5_v6_large/final_model\"\n# model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n# tokenizer = AutoTokenizer.from_pretrained(model_path)\n# print(\"✅ 加载完成\\n\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-01T12:40:54.024885Z","iopub.execute_input":"2026-03-01T12:40:54.025256Z","iopub.status.idle":"2026-03-01T12:40:54.029172Z","shell.execute_reply.started":"2026-03-01T12:40:54.025224Z","shell.execute_reply":"2026-03-01T12:40:54.028467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # 测试样本\n# test_cases = [\n#     \"translate Akkadian to English: 14 ma-na KÙ.BABBAR ṣa-ru-pá-am i-na ší-im SÍG.HI.A ša ú-ší-na-lam bu-zu-ta-a il5-qé\",\n# ]\n\n# # 测试推理\n# for i, text in enumerate(test_cases, 1):\n#     print(f\"{'='*70}\")\n#     print(f\"测试 {i}:\")\n#     print(f\"{'='*70}\")\n#     print(f\"输入: {text}\")\n    \n#     inputs = tokenizer(text, return_tensors=\"pt\", max_length=768, truncation=True)\n#     outputs = model.generate(**inputs, max_length=256, num_beams=4, early_stopping=True)\n#     result = tokenizer.decode(outputs[0], skip_special_tokens=True)\n    \n#     print(f\"输出: {result}\")\n    \n#     # 判断\n#     if result == text:\n#         print(\"❌ 输出=输入（copy模式）\")\n#     elif \"{\" in result or \"sze\" in result or \"szar\" in result:\n#         print(\"❌ 输出还是 Akkadian\")\n#     else:\n#         print(\"✅ 输出看起来像 English\")\n    \n#     print()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-01T12:40:54.030481Z","iopub.execute_input":"2026-03-01T12:40:54.030838Z","iopub.status.idle":"2026-03-01T12:40:54.040417Z","shell.execute_reply.started":"2026-03-01T12:40:54.030812Z","shell.execute_reply":"2026-03-01T12:40:54.039910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nKaggle Submission Pipeline (v1_improved_no_tag)\n功能：加载测试集 -> 预处理 -> 批量推理 -> 后处理 -> 生成 submission.csv\n\"\"\"\nimport os\nimport sys\nimport subprocess\nimport torch\nimport pandas as pd\nimport re\nimport json\nfrom tqdm import tqdm\nfrom typing import List, Dict\n\n# ============ 动态加载清洗模块（兼容 Kaggle 和本地环境） ============\ndef load_cleaning_module():\n    \"\"\"自动检测环境并加载 clean_akkadian, clean_english\"\"\"\n    # 方式1: Kaggle Dataset 路径\n    kaggle_paths = [\n        '/kaggle/input/datasets/wukeneth/text-normalization/text_normalization.py',\n    ]\n    for kp in kaggle_paths:\n        if os.path.exists(kp):\n            # 直接导入\n            import importlib.util\n            spec = importlib.util.spec_from_file_location(\"text_normalization\", kp)\n            module = importlib.util.module_from_spec(spec)\n            spec.loader.exec_module(module)\n            print(f\"✅ 从 Kaggle 加载清洗模块: {kp}\")\n            return module.clean_akkadian, module.clean_english\n\n    # 方式2: 硬编码本地路径\n    local_root = \"D:/py_projects/translation/sentence_breaker\"\n    local_path = local_root + \"/core/text/text_normalization.py\"\n    if os.path.exists(local_path):\n        if local_root not in sys.path:\n            sys.path.insert(0, local_root)\n        from core.text.text_normalization import clean_akkadian, clean_english\n        print(f\"✅ 从本地加载清洗模块: {local_root}\")\n        return clean_akkadian, clean_english\n\n    # 如果都找不到，抛出错误\n    raise FileNotFoundError(\n        \"❌ 无法找到 text_normalization 模块！\\n\"\n        \"请确保在 Kaggle 中添加 text-normalization 数据集，\"\n        \"或在本地环境中确保项目路径正确。\"\n    )\n\n# 加载清洗函数\nclean_akkadian, clean_english = load_cleaning_module()\n\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n# ============ 配置区 ============\nRUN_EVALUATION = False  # True: 跑验证集; False: 跑测试集\n\n# 验证集路径\nEVAL_DATASET_PATHS = [\n    \"./datasets_v1/split/eval.jsonl\",\n]\n\n# 模型路径 (Kaggle Dataset 本地路径)\nMODEL_PATH = \"/kaggle/input/datasets/wukeneth/byt5-v9-model-last/step_2647/final_model\"\n\n# 测试集路径\nTEST_FILE = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n\n# 输出文件\nOUTPUT_FILE = \"submission.csv\"\n\n# 推理参数\nBATCH_SIZE = 16\nNUM_BEAMS = 4\nMAX_LENGTH = 768\nREPETITION_PENALTY = 1.2\nLENGTH_PENALTY = 1.0\n# =============================================\n\n# 自动安装 sacrebleu\nif RUN_EVALUATION:\n    try:\n        import sacrebleu\n    except ImportError:\n        print(\"📦 正在安装 sacrebleu...\")\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"sacrebleu\"])\n        import sacrebleu\n\n\n# ------------- 核心流程 -------------\n\n\ndef load_test_data(path):\n    print(f\"📖 加载测试集: {path}\")\n    df = pd.read_csv(path)\n    if 'transliteration' not in df.columns:\n        if 'text' in df.columns:\n            df['transliteration'] = df['text']\n        elif 'akkadian' in df.columns:\n            df['transliteration'] = df['akkadian']\n    print(f\"📊 测试集大小: {len(df)}\")\n    return df\n\n\ndef preprocess_data(texts):\n    print(\"🧹 正在预处理数据...\")\n    processed = []\n    for text in tqdm(texts, desc=\"Preprocessing\"):\n        cleaned = clean_akkadian(text)\n        final_input = f\"translate Akkadian to English: {cleaned}\"\n        processed.append(final_input)\n    return processed\n\n\ndef batch_inference(model, tokenizer, texts, device):\n    print(f\"⚡ 开始分桶批量推理 (Batch Size={BATCH_SIZE})...\")\n    model.eval()\n\n    # 按长度排序，记录原始顺序\n    indexed = sorted(enumerate(texts), key=lambda x: len(x[1]))\n    original_indices = [i for i, _ in indexed]\n    sorted_texts = [t for _, t in indexed]\n\n    sorted_preds = []\n    for i in tqdm(range(0, len(sorted_texts), BATCH_SIZE), desc=\"Inference\"):\n        batch_texts = sorted_texts[i:i + BATCH_SIZE]\n        inputs = tokenizer(\n                batch_texts,\n                return_tensors=\"pt\",\n                truncation=True,\n                max_length=MAX_LENGTH,\n                padding=True\n            ).to(device)\n\n        with torch.no_grad():\n            outputs = model.generate(\n                inputs.input_ids,\n                attention_mask=inputs.attention_mask,\n                max_length=MAX_LENGTH,\n                num_beams=NUM_BEAMS,\n                repetition_penalty=REPETITION_PENALTY,\n                length_penalty=LENGTH_PENALTY,\n                early_stopping=True\n            )\n        preds = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        sorted_preds.extend([p.strip() for p in preds])\n\n    # 还原原始顺序\n    all_preds = [None] * len(texts)\n    for orig_idx, pred in zip(original_indices, sorted_preds):\n        all_preds[orig_idx] = pred\n\n    return all_preds\n\n\ndef main():\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    print(f\"⚙️ 运行设备: {device}\")\n\n    # 1. 加载模型\n    print(f\"🚀 加载模型: {MODEL_PATH}\")\n    try:\n        # ByT5 tokenizer 是固定字节编码，无需从文件加载，直接实例化\n        from transformers.models.byt5.tokenization_byt5 import ByT5Tokenizer\n        tokenizer = ByT5Tokenizer()\n        model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH).to(device)\n    except Exception as e:\n        import traceback\n        print(f\"❌ 模型加载失败: {e}\")\n        traceback.print_exc()\n        return\n\n    # 2. 确定运行模式并加载数据\n    if RUN_EVALUATION:\n        print(\"\\n📊 === 进入评估模式 ===\")\n        eval_path = None\n        for p in EVAL_DATASET_PATHS:\n            if os.path.exists(p):\n                eval_path = p\n                break\n\n        if not eval_path:\n            print(f\"❌ 找不到验证集文件\")\n            return\n\n        eval_data = []\n        with open(eval_path, 'r', encoding='utf-8') as f:\n            for line in f:\n                if line.strip():\n                    eval_data.append(json.loads(line))\n\n        # 提取 source 和 target\n        df = pd.DataFrame(eval_data)\n        if 'source' in df.columns:\n            df['transliteration'] = df['source'].apply(\n                lambda x: x.replace(\"translate Akkadian to English: \", \"\").strip()\n            )\n        if 'target' in df.columns:\n            df['ground_truth'] = df['target']\n\n        print(f\"📊 验证集大小: {len(df)}\")\n        target_col = 'ground_truth'\n    else:\n        print(\"\\n🚀 === 进入提交模式 ===\")\n        test_df = load_test_data(TEST_FILE)\n        df = test_df\n        target_col = None\n\n    # 3. 预处理 (构造模型输入)\n    if RUN_EVALUATION:\n        # 验证集模式：直接使用 source（已包含完整指令），无需清洗\n        if 'source' in df.columns and \"translate Akkadian to English: \" in df['source'].iloc[0]:\n            print(\"ℹ️ 验证集已包含指令前缀，直接使用。\")\n            inputs = df['source'].tolist()\n        else:\n            inputs = preprocess_data(df['transliteration'].tolist())\n    else:\n        inputs = preprocess_data(df['transliteration'].tolist())\n\n    raw_texts = df['transliteration'].tolist()\n\n    # 4. 推理\n    predictions = batch_inference(model, tokenizer, inputs, device)\n\n    # 5. 后处理\n    print(\"🔧 正在进行后处理...\")\n    final_preds = []\n    for pred in tqdm(predictions, desc=\"Post-processing\"):\n        if not pred or str(pred).strip() == \"\" or pred == \"(error_oom)\":\n            pred = \"(empty_line)\"\n        final_preds.append(pred)\n\n    # 6. 分支\n    if RUN_EVALUATION:\n        print(\"\\n📈 计算评估指标...\")\n        from sacrebleu.metrics import BLEU, CHRF\n\n        refs = df[target_col].tolist()\n\n        bleu = BLEU()\n        chrf = CHRF()\n        b_score = bleu.corpus_score(final_preds, [refs]).score\n        c_score = chrf.corpus_score(final_preds, [refs]).score\n        combo = (b_score * c_score) ** 0.5\n\n        print(f\"BLEU:  {b_score:.2f}\")\n        print(f\"chrF:  {c_score:.2f}\")\n        print(f\"Combo: {combo:.2f}\")\n    else:\n        # 生成 submission\n        submission = pd.DataFrame({\n            'id': df['id'],\n            'translation': final_preds\n        })\n        submission.to_csv(OUTPUT_FILE, index=False)\n        print(f\"✅ Submission 已保存至: {OUTPUT_FILE}\")\n        print(\"\\n🔍 预测示例:\")\n        for i in range(min(5, len(final_preds))):\n            print(f\"  [{i}] {final_preds[i][:120]}\")\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-01T15:52:59.585432Z","iopub.execute_input":"2026-03-01T15:52:59.586076Z"}},"outputs":[],"execution_count":null}]}