{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"}},"cells":[{"cell_type":"markdown","id":"md-title","metadata":{},"source":"# Deep Past — Akkadian → English 推論 (MBR デコーディング)\n\n## 方針\n- **モデル**: `mattiaangeli/byt5-akkadian-mbr`（アッカド語翻訳専用fine-tune済み）\n- **デコード**: MBR (Minimum Bayes Risk) — beam候補 + サンプリング候補をchrF++で選択\n- **評価指標**: BLEU × chrF++ の幾何平均"},{"cell_type":"code","execution_count":null,"id":"cell-imports","metadata":{},"outputs":[],"source":"import os\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom pathlib import Path\nfrom contextlib import nullcontext\nfrom typing import List\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport sacrebleu\n\nprint(f'PyTorch: {torch.__version__}')\nprint(f'CUDA available: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')\n    total_gb = torch.cuda.get_device_properties(0).total_memory / 1e9\n    print(f'GPU Memory: {total_gb:.1f} GB')"},{"cell_type":"code","execution_count":null,"id":"cell-config","metadata":{},"outputs":[],"source":"# ─── 設定 ───────────────────────────────────────────────────────\nDATA_DIR   = Path('/kaggle/input/deep-past-initiative-machine-translation')\nMODEL_PATH = '/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr/pytorch/default/6'\nOUTPUT_DIR = Path('/kaggle/working')\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# 生成パラメータ\nMAX_LENGTH     = 256\nMAX_NEW_TOKENS = 192\nNUM_BEAMS      = 8\nLENGTH_PENALTY = 1.2\nBATCH_SIZE     = 4\nNUM_WORKERS    = 2\n\n# MBR パラメータ\nMBR_BEAM_CANDS   = 4   # beam search で生成する候補数\nMBR_SAMPLE_CANDS = 4   # sampling で生成する候補数\nMBR_TOP_P        = 0.9\nMBR_TEMPERATURE  = 0.8\n\n# BF16 サポート確認\nBF16_SUPPORTED = (\n    DEVICE.type == 'cuda' and\n    getattr(torch.cuda, 'is_bf16_supported', lambda: False)()\n)\nprint(f'BF16 supported: {BF16_SUPPORTED}')\nprint(f'Device: {DEVICE}')"},{"cell_type":"code","execution_count":null,"id":"cell-model-load","metadata":{},"outputs":[],"source":"# ─── モデル・トークナイザーのロード ────────────────────────────\nprint(f'Loading model from {MODEL_PATH} ...')\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\nmodel     = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH, local_files_only=True)\nmodel     = model.to(DEVICE).eval()\n\nif DEVICE.type == 'cuda':\n    try:\n        torch.set_float32_matmul_precision('high')\n    except Exception:\n        pass\n\n# BetterTransformer で推論を高速化（使えれば）\ntry:\n    from optimum.bettertransformer import BetterTransformer\n    model = BetterTransformer.transform(model)\n    print('BetterTransformer applied')\nexcept Exception as e:\n    print(f'BetterTransformer skipped: {e}')\n\nn_params = sum(p.numel() for p in model.parameters()) / 1e6\nprint(f'Model loaded: {n_params:.0f}M parameters')"},{"cell_type":"code","execution_count":null,"id":"cell-preprocess","metadata":{},"outputs":[],"source":"# ─── テキスト前処理 ─────────────────────────────────────────────\n# 欠損マーカー（...や xx）を特殊トークンに変換する\n_RE_BIG_GAP   = re.compile(r'(\\.{3,}|…+)')\n_RE_SMALL_GAP = re.compile(r'(xx+|\\s+x\\s+)')\n\nPREFIX = 'translate Akkadian to English: '\n\ndef preprocess(text: str) -> str:\n    if pd.isna(text):\n        return ''\n    t = str(text)\n    t = _RE_BIG_GAP.sub('<big_gap>', t)\n    t = _RE_SMALL_GAP.sub('<gap>', t)\n    return t\n\ndef preprocess_batch(texts: List[str]) -> List[str]:\n    s = pd.Series(texts).fillna('').astype(str)\n    s = s.str.replace(_RE_BIG_GAP, '<big_gap>', regex=True)\n    s = s.str.replace(_RE_SMALL_GAP, '<gap>', regex=True)\n    return s.tolist()\n\nprint('Preprocessor ready')"},{"cell_type":"code","execution_count":null,"id":"cell-postprocess","metadata":{},"outputs":[],"source":"# ─── テキスト後処理 ─────────────────────────────────────────────\n_RE_GAP          = re.compile(r'(\\[x\\]|\\(x\\)|\\bx\\b)', re.I)\n_RE_BIG_G        = re.compile(r'(\\.{3,}|…|\\[\\.+\\])')\n_RE_ANNOT        = re.compile(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..*?\\)', re.I)\n_RE_REPEAT_WORD  = re.compile(r'\\b(\\w+)(?:\\s+\\1\\b)+')\n_RE_WS           = re.compile(r'\\s+')\n_RE_PUNCT_SP     = re.compile(r'\\s+([.,:])')\n_RE_REPEAT_PUNCT = re.compile(r'([.,])\\1+')\n\n_SUBSCRIPT = str.maketrans('₀₁₂₃₄₅₆₇₈₉', '0123456789')\n_SPECIAL   = str.maketrans('ḫḪ', 'hH')\n_FORBIDDEN = str.maketrans('', '', '!?(\"——<>⌈⌋⌊[]+ʾ/;')\n\ndef postprocess_batch(texts: List[str]) -> List[str]:\n    s = pd.Series(texts).fillna('').astype(str)\n\n    # 文字変換\n    s = s.str.translate(_SPECIAL)\n    s = s.str.translate(_SUBSCRIPT)\n\n    # gap の正規化\n    s = s.str.replace(_RE_GAP, '<gap>', regex=True)\n    s = s.str.replace(_RE_BIG_G, '<big_gap>', regex=True)\n    s = s.str.replace('<gap> <gap>', '<big_gap>', regex=False)\n    s = s.str.replace('<big_gap> <big_gap>', '<big_gap>', regex=False)\n\n    # 注釈除去\n    s = s.str.replace(_RE_ANNOT, '', regex=True)\n\n    # gap を一時保護してから禁止文字を除去\n    s = s.str.replace('<gap>', '\\x00GAP\\x00', regex=False)\n    s = s.str.replace('<big_gap>', '\\x00BIG\\x00', regex=False)\n    s = s.str.translate(_FORBIDDEN)\n    s = s.str.replace('\\x00GAP\\x00', ' <gap> ', regex=False)\n    s = s.str.replace('\\x00BIG\\x00', ' <big_gap> ', regex=False)\n\n    # 分数表記\n    s = s.str.replace(r'(\\d+)\\.5\\b', r'\\1½', regex=True)\n    s = s.str.replace(r'\\b0\\.5\\b',   '½',    regex=True)\n    s = s.str.replace(r'(\\d+)\\.25\\b', r'\\1¼', regex=True)\n    s = s.str.replace(r'\\b0\\.25\\b',  '¼',    regex=True)\n\n    # 重複語・句読点の整理\n    s = s.str.replace(_RE_REPEAT_WORD, r'\\1', regex=True)\n    s = s.str.replace(_RE_PUNCT_SP,    r'\\1', regex=True)\n    s = s.str.replace(_RE_REPEAT_PUNCT, r'\\1', regex=True)\n\n    # 最終クリーンアップ\n    s = s.str.replace(_RE_WS, ' ', regex=True)\n    s = s.str.strip().str.strip('-').str.strip()\n\n    return s.tolist()\n\nprint('Postprocessor ready')"},{"cell_type":"code","execution_count":null,"id":"cell-mbr","metadata":{},"outputs":[],"source":"# ─── MBR (Minimum Bayes Risk) デコーディング ───────────────────\n# 複数の翻訳候補を生成し、chrF++類似度が最大の候補を選ぶ\n\n_CHRFPP = sacrebleu.metrics.CHRF(word_order=2)\n\ndef _chrfpp_sim(a: str, b: str) -> float:\n    a, b = (a or '').strip(), (b or '').strip()\n    if not a or not b:\n        return 0.0\n    return float(_CHRFPP.sentence_score(a, [b]).score)\n\ndef mbr_pick(candidates: List[str]) -> str:\n    \"\"\"候補群の中から平均chrF++が最大の翻訳を選ぶ\"\"\"\n    # 重複排除\n    seen, unique = set(), []\n    for c in candidates:\n        c = str(c).strip()\n        if c and c not in seen:\n            unique.append(c)\n            seen.add(c)\n\n    if not unique:\n        return ''\n    if len(unique) == 1:\n        return unique[0]\n\n    best_i, best_score = 0, -1e9\n    n = len(unique)\n    for i in range(n):\n        score = sum(_chrfpp_sim(unique[i], unique[j]) for j in range(n) if j != i) / (n - 1)\n        if score > best_score:\n            best_score, best_i = score, i\n\n    return unique[best_i]\n\nprint('MBR ready')"},{"cell_type":"code","execution_count":null,"id":"cell-generate","metadata":{},"outputs":[],"source":"# ─── バッチ生成 (Beam + Sampling → MBR) ────────────────────────\n\ndef generate_mbr_batch(\n    input_ids: torch.Tensor,\n    attention_mask: torch.Tensor,\n) -> List[str]:\n    B = input_ids.shape[0]\n    gen_base = dict(max_new_tokens=MAX_NEW_TOKENS, repetition_penalty=1.2, use_cache=True)\n\n    # ── Beam search 候補 ──\n    beam_out = model.generate(\n        input_ids=input_ids,\n        attention_mask=attention_mask,\n        do_sample=False,\n        num_beams=max(NUM_BEAMS, MBR_BEAM_CANDS),\n        num_return_sequences=MBR_BEAM_CANDS,\n        length_penalty=LENGTH_PENALTY,\n        early_stopping=True,\n        **gen_base,\n    )\n    beam_texts = tokenizer.batch_decode(beam_out, skip_special_tokens=True)\n\n    # B × MBR_BEAM_CANDS の候補プール\n    pools = [list(beam_texts[i * MBR_BEAM_CANDS:(i + 1) * MBR_BEAM_CANDS]) for i in range(B)]\n\n    # ── Sampling 候補 ──\n    if MBR_SAMPLE_CANDS > 0:\n        samp_out = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            do_sample=True,\n            num_beams=1,\n            top_p=MBR_TOP_P,\n            temperature=MBR_TEMPERATURE,\n            num_return_sequences=MBR_SAMPLE_CANDS,\n            **gen_base,\n        )\n        samp_texts = tokenizer.batch_decode(samp_out, skip_special_tokens=True)\n        for i in range(B):\n            pools[i].extend(samp_texts[i * MBR_SAMPLE_CANDS:(i + 1) * MBR_SAMPLE_CANDS])\n\n    return [mbr_pick(pool) for pool in pools]\n\nprint('Generator ready')"},{"cell_type":"code","execution_count":null,"id":"cell-dataset","metadata":{},"outputs":[],"source":"# ─── データセットの読み込み ──────────────────────────────────────\ntest = pd.read_csv(DATA_DIR / 'test.csv', encoding='utf-8')\nprint(f'テストデータ: {len(test)} 件')\nprint(test.head(3))\n\nclass AkkadianDataset(Dataset):\n    def __init__(self, df: pd.DataFrame):\n        self.ids   = df['id'].tolist()\n        raw        = df['transliteration'].tolist()\n        cleaned    = preprocess_batch(raw)\n        self.texts = [PREFIX + t for t in cleaned]\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        return self.ids[idx], self.texts[idx]\n\ndef collate_fn(batch):\n    ids   = [b[0] for b in batch]\n    texts = [b[1] for b in batch]\n    enc   = tokenizer(texts, max_length=MAX_LENGTH, padding=True,\n                      truncation=True, return_tensors='pt')\n    return ids, enc\n\ndataset    = AkkadianDataset(test)\ndataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False,\n                        num_workers=NUM_WORKERS, collate_fn=collate_fn,\n                        pin_memory=True)\nprint(f'Batches: {len(dataloader)}')"},{"cell_type":"code","execution_count":null,"id":"cell-inference","metadata":{},"outputs":[],"source":"# ─── 推論ループ ─────────────────────────────────────────────────\nautocast_ctx = (\n    torch.autocast(device_type='cuda', dtype=torch.bfloat16)\n    if BF16_SUPPORTED else nullcontext()\n)\n\nresults = []\n\nwith torch.inference_mode(), autocast_ctx:\n    for batch_ids, enc in tqdm(dataloader, desc='Translating'):\n        input_ids      = enc.input_ids.to(DEVICE)\n        attention_mask = enc.attention_mask.to(DEVICE)\n\n        raw_preds = generate_mbr_batch(input_ids, attention_mask)\n        cleaned   = postprocess_batch(raw_preds)\n\n        results.extend(zip(batch_ids, cleaned))\n\n        if DEVICE.type == 'cuda':\n            torch.cuda.empty_cache()\n\nprint(f'\\n推論完了: {len(results)} 件')"},{"cell_type":"code","execution_count":null,"id":"cell-save","metadata":{},"outputs":[],"source":"# ─── submission.csv の保存 ─────────────────────────────────────\nsubmission = pd.DataFrame(results, columns=['id', 'translation'])\nsubmission.to_csv(OUTPUT_DIR / 'submission.csv', index=False)\n\nprint(f'submission.csv 保存完了 ({len(submission)} 行)')\nprint(f'空の翻訳: {submission[\"translation\"].str.strip().eq(\"\").sum()} 件')\nprint('\\n--- サンプル ---')\nfor _, row in submission.head(5).iterrows():\n    print(f'  id={row[\"id\"]}: {str(row[\"translation\"])[:100]}')\n\nsubmission"}]}