{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14497020,"datasetId":9259464,"databundleVersionId":15322265},{"sourceType":"kernelVersion","sourceId":293293667}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-09T09:59:21.520367Z","iopub.execute_input":"2026-03-09T09:59:21.521319Z","iopub.status.idle":"2026-03-09T09:59:21.538450Z","shell.execute_reply.started":"2026-03-09T09:59:21.521276Z","shell.execute_reply":"2026-03-09T09:59:21.537875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\n\n# 环境设置\nos.environ[\"TQDM_DISABLE\"] = \"0\"\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# ============================================================\n# CONFIGURATION\n# ============================================================\nCONFIG = {\n \"model_path\": \"/kaggle/input/notebooks/dylan12138/add-newdataset1-2-gpt-0118/byt5_akkadian_model_with_dict\",\n \"test_path\": \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\",\n \"dict_path\": \"/kaggle/input/datasets/dylan12138/newdata2-0114/new_dataset2_0112_processed.xlsx\",\n \"batch_size\": 16, \n \"max_length\": 512,\n \"num_beams\": 5, \n \"early_stopping\": True,\n \"use_host_normalization\": True,\n \"repetition_penalty\": 1.2,\n \"length_penalty\": 1.0,\n}\n\n# ============================================================\n# DICTIONARY MODULE\n# ============================================================\nclass AkkadianWordMatcher:\n def __init__(self, excel_path):\n print(f\"Loading dictionary: {excel_path}\")\n df = pd.read_excel(excel_path)\n self.word_map = {}\n for _, row in df.iterrows():\n akk_w = str(row['input_text']).strip()\n # 处理多义词，只取第一个\n eng_w = str(row['target_text']).split('|')[0].split('丨')[0].strip()\n if akk_w and eng_w:\n self.word_map[akk_w] = eng_w\n \n self.sorted_akk = sorted(self.word_map.keys(), key=len, reverse=True)\n self.ak_chars = r\"a-zA-Z0-9\\.\\u00C0-\\u017F\\u2080-\\u2089\"\n\n def replace(self, text):\n new_text = text.replace('+', '-') \n for akk in self.sorted_akk:\n pattern = re.compile(rf'(? str:\n result = text\n for k, v in ASCII_TO_DIACRITIC.items(): result = result.replace(k, v)\n for k, v in VOWEL_SUBSCRIPTS.items(): result = result.replace(k, v)\n \n # 简单的 Gap 处理\n result = re.sub(r'\\bx\\b', '', result)\n result = re.sub(r'(\\b\\s*){2,}', ' ', result)\n return result.strip()\n\n# ============================================================\n# PREPROCESSING PIPELINE\n# ============================================================\nword_matcher = AkkadianWordMatcher(CONFIG[\"dict_path\"])\nPREFIX = \"translate Akkadian to English: \"\n\ndef preprocess(text: str) -> str:\n if CONFIG[\"use_host_normalization\"]:\n text = apply_host_normalizations(text)\n # 词典替换：将阿卡德词直接替换为对应的英文锚点\n text = word_matcher.replace(text)\n return PREFIX + text\n\n# ============================================================\n# DATA & MODEL LOADING\n# ============================================================\nprint(f\"Loading model and data...\")\ntokenizer = AutoTokenizer.from_pretrained(CONFIG[\"model_path\"])\nmodel = AutoModelForSeq2SeqLM.from_pretrained(CONFIG[\"model_path\"]).to(DEVICE)\nmodel.eval()\n\ntest_df = pd.read_csv(CONFIG[\"test_path\"])\n\nclass InferenceDataset(Dataset):\n def __init__(self, df, tokenizer):\n # 核心：在创建 Dataset 时预处理所有文本\n self.texts = [preprocess(str(t)) for t in df[\"transliteration\"].tolist()]\n self.tokenizer = tokenizer\n def __len__(self): return len(self.texts)\n def __getitem__(self, idx):\n return self.tokenizer(self.texts[idx], max_length=CONFIG[\"max_length\"], \n padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n\ntest_loader = DataLoader(InferenceDataset(test_df, tokenizer), batch_size=CONFIG[\"batch_size\"])\n\n# ============================================================\n# INFERENCE LOOP\n# ============================================================\nprint(\"\\n\" + \"=\" * 60)\nprint(\"GENERATING TRANSLATIONS\")\nprint(\"=\" * 60)\n\nall_predictions = []\nwith torch.no_grad():\n for batch in tqdm(test_loader, desc=\"Inference\"):\n input_ids = batch[\"input_ids\"].squeeze(1).to(DEVICE)\n attention_mask = batch[\"attention_mask\"].squeeze(1).to(DEVICE)\n\n outputs = model.generate(\n input_ids=input_ids,\n attention_mask=attention_mask,\n max_length=CONFIG[\"max_length\"],\n num_beams=CONFIG[\"num_beams\"],\n repetition_penalty=CONFIG[\"repetition_penalty\"],\n length_penalty=CONFIG[\"length_penalty\"],\n early_stopping=True\n )\n decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n all_predictions.extend([d.strip() for d in decoded])\n\n# ============================================================\n# FINAL SUBMISSION\n# ============================================================\nsubmission = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": all_predictions})\nsubmission[\"translation\"] = submission[\"translation\"].fillna(\"broken text\").apply(lambda x: x if len(x)>0 else \"broken text\")\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(f\"\\nSaved {len(submission)} rows. Head:\\n\", submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-09T10:02:35.985244Z","iopub.execute_input":"2026-03-09T10:02:35.985844Z","iopub.status.idle":"2026-03-09T10:02:45.742465Z","shell.execute_reply.started":"2026-03-09T10:02:35.985808Z","shell.execute_reply":"2026-03-09T10:02:45.741820Z"}},"outputs":[],"execution_count":null}]}