{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14197093,"sourceType":"datasetVersion","datasetId":9053011},{"sourceId":14231981,"sourceType":"datasetVersion","datasetId":9079497},{"sourceId":287248975,"sourceType":"kernelVersion"},{"sourceId":287306416,"sourceType":"kernelVersion"},{"sourceId":288281790,"sourceType":"kernelVersion"},{"sourceId":282751,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":239470,"modelId":222398},{"sourceId":690442,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":523415,"modelId":537419},{"sourceId":690795,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":523698,"modelId":537719},{"sourceId":691388,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":524188,"modelId":538201},{"sourceId":692580,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":525132,"modelId":539167}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM, Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq\nfrom tqdm.auto import tqdm\n\nDATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\nBASE_MODEL = \"/kaggle/input/byt5-base/transformers/default/1/byt5-base-saved\"\nif not os.path.exists(BASE_MODEL):\n    BASE_MODEL = \"google/byt5-base\"\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nMAX_LENGTH = 256  # Reduced for speed\nMAX_PAIRS = 5000  # Cap data for speed\nprint(f\"Device: {DEVICE}, Model: {BASE_MODEL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T10:15:59.483620Z","iopub.execute_input":"2026-01-04T10:15:59.483830Z","iopub.status.idle":"2026-01-04T10:16:31.962439Z","shell.execute_reply.started":"2026-01-04T10:15:59.483809Z","shell.execute_reply":"2026-01-04T10:16:31.961774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalization function\ndef normalize_text(text):\n    if pd.isna(text) or not isinstance(text, str):\n        return \"\"\n    text = str(text)\n    text = text.replace('á', 'a2').replace('à', 'a3')\n    text = text.replace('é', 'e2').replace('è', 'e3')\n    text = text.replace('í', 'i2').replace('ì', 'i3')\n    text = text.replace('ú', 'u2').replace('ù', 'u3')\n    subscripts = {'₀':'0','₁':'1','₂':'2','₃':'3','₄':'4','₅':'5','₆':'6','₇':'7','₈':'8','₉':'9','ₓ':'x'}\n    for sub, num in subscripts.items():\n        text = text.replace(sub, num)\n    text = text.replace('(d)', '{d}').replace('(D)', '{d}')\n    text = re.sub(r'[!?/]', '', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef clean_translation(text):\n    if pd.isna(text) or not isinstance(text, str):\n        return \"\"\n    return re.sub(r'\\s+', ' ', str(text)).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T10:16:31.963829Z","iopub.execute_input":"2026-01-04T10:16:31.964362Z","iopub.status.idle":"2026-01-04T10:16:31.970992Z","shell.execute_reply.started":"2026-01-04T10:16:31.964334Z","shell.execute_reply":"2026-01-04T10:16:31.970453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and extract data\nprint(\"Loading data...\")\ntrain_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\npub_df = pd.read_csv(f\"{DATA_DIR}/published_texts.csv\")\n\nall_pairs = []\n\n# Train.csv\nfor _, row in train_df.iterrows():\n    t, e = row.get('transliteration', ''), row.get('translation', '')\n    if pd.notna(t) and pd.notna(e) and len(str(t)) > 5 and len(str(e)) > 5:\n        all_pairs.append({'transliteration': normalize_text(t), 'translation': clean_translation(e), 'source': 'train'})\n\nprint(f\"From train.csv: {len(all_pairs)}\")\n\n# Published_texts.csv - use AICC_translation!\nt_col = 'transliteration'\ne_col = 'AICC_translation' if 'AICC_translation' in pub_df.columns else 'translation'\n\nif t_col in pub_df.columns and e_col in pub_df.columns:\n    pub_count = 0\n    for _, row in pub_df.iterrows():\n        t, e = row[t_col], row[e_col]\n        if pd.notna(t) and pd.notna(e) and len(str(t)) > 10 and len(str(e)) > 5:\n            all_pairs.append({'transliteration': normalize_text(str(t)), 'translation': clean_translation(str(e)), 'source': 'pub'})\n            pub_count += 1\n            if len(all_pairs) >= MAX_PAIRS:\n                break\n    print(f\"From published_texts: {pub_count}\")\n\n# Cap and deduplicate\ndf_all = pd.DataFrame(all_pairs[:MAX_PAIRS])\ndf_all = df_all.drop_duplicates(subset=['transliteration'])\nprint(f\"TOTAL: {len(df_all)} pairs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T10:16:31.974227Z","iopub.execute_input":"2026-01-04T10:16:31.974614Z","iopub.status.idle":"2026-01-04T10:16:32.832732Z","shell.execute_reply.started":"2026-01-04T10:16:31.974592Z","shell.execute_reply":"2026-01-04T10:16:32.831964Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset\nclass AkkadianDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.df = df.reset_index(drop=True)\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        source = \"translate Akkadian to English: \" + str(row['transliteration'])\n        target = str(row['translation'])\n        \n        src = self.tokenizer(source, max_length=MAX_LENGTH, padding='max_length', truncation=True, return_tensors='pt')\n        with self.tokenizer.as_target_tokenizer():\n            tgt = self.tokenizer(target, max_length=MAX_LENGTH, padding='max_length', truncation=True, return_tensors='pt')\n        \n        labels = tgt['input_ids'].squeeze()\n        labels[labels == self.tokenizer.pad_token_id] = -100\n        \n        return {'input_ids': src['input_ids'].squeeze(), 'attention_mask': src['attention_mask'].squeeze(), 'labels': labels}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T10:16:32.833761Z","iopub.execute_input":"2026-01-04T10:16:32.834170Z","iopub.status.idle":"2026-01-04T10:16:32.839858Z","shell.execute_reply.started":"2026-01-04T10:16:32.834135Z","shell.execute_reply":"2026-01-04T10:16:32.839238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training\nprint(\"Loading model...\")\ntokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(BASE_MODEL)\n\ntrain_dataset = AkkadianDataset(df_all, tokenizer)\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian_model\",\n    num_train_epochs=2,  # FAST: only 2 epochs\n    per_device_train_batch_size=4,  # Larger batch\n    gradient_accumulation_steps=4,  # Effective batch = 16\n    learning_rate=5e-4,  # Higher LR for fewer epochs\n    warmup_steps=50,\n    logging_steps=100,\n    save_strategy=\"no\",  # Don't save checkpoints\n    fp16=True,\n    gradient_checkpointing=True,\n    optim=\"adafactor\",\n    report_to=\"none\",\n    dataloader_num_workers=0,\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    data_collator=DataCollatorForSeq2Seq(tokenizer, model=model),\n)\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nprint(\"Training...\")\ntrainer.train()\nprint(\"Done!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T10:16:35.492442Z","iopub.execute_input":"2026-01-04T10:16:35.492962Z","iopub.status.idle":"2026-01-04T14:52:03.369898Z","shell.execute_reply.started":"2026-01-04T10:16:35.492933Z","shell.execute_reply":"2026-01-04T14:52:03.368860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference\ntest_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\ntest_df['transliteration_norm'] = test_df['transliteration'].apply(normalize_text)\n\nmodel.eval()\npredictions = []\n\nprint(\"Inference...\")\nwith torch.no_grad():\n    for _, row in tqdm(test_df.iterrows(), total=len(test_df)):\n        text = \"translate Akkadian to English: \" + str(row['transliteration_norm'])\n        inputs = tokenizer(text, return_tensors='pt', max_length=MAX_LENGTH, truncation=True).to(DEVICE)\n        outputs = model.generate(**inputs, max_length=MAX_LENGTH, num_beams=4, early_stopping=True)\n        predictions.append(tokenizer.decode(outputs[0], skip_special_tokens=True).strip())\n\n# Post-process\ndef post_process(t):\n    if not t or len(t) < 2: return \"broken text\"\n    t = re.sub(r'\\bkanesh\\b', 'Kanesh', t, flags=re.IGNORECASE)\n    t = re.sub(r'\\bassur\\b', 'Assur', t, flags=re.IGNORECASE)\n    return t.strip()\n\npredictions = [post_process(p) for p in predictions]\n\npd.DataFrame({'id': test_df['id'], 'translation': predictions}).to_csv('submission.csv', index=False)\nprint(\"Saved submission.csv!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T14:52:03.371729Z","iopub.execute_input":"2026-01-04T14:52:03.372063Z","iopub.status.idle":"2026-01-04T14:52:05.984689Z","shell.execute_reply.started":"2026-01-04T14:52:03.372029Z","shell.execute_reply":"2026-01-04T14:52:05.983987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}