{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\nprint(train.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:38:37.809155Z","iopub.execute_input":"2026-02-01T10:38:37.809532Z","iopub.status.idle":"2026-02-01T10:38:38.304788Z","shell.execute_reply.started":"2026-02-01T10:38:37.809505Z","shell.execute_reply":"2026-02-01T10:38:38.303479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:38:57.000321Z","iopub.execute_input":"2026-02-01T10:38:57.000663Z","iopub.status.idle":"2026-02-01T10:38:57.010088Z","shell.execute_reply.started":"2026-02-01T10:38:57.000633Z","shell.execute_reply":"2026-02-01T10:38:57.009030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:40:17.273032Z","iopub.execute_input":"2026-02-01T10:40:17.273420Z","iopub.status.idle":"2026-02-01T10:40:17.280091Z","shell.execute_reply.started":"2026-02-01T10:40:17.273389Z","shell.execute_reply":"2026-02-01T10:40:17.279108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport unicodedata\n\n############################################\n# 1. BASIC CHARACTER NORMALIZATION\n############################################\n\ndef normalize_unicode(text):\n    \"\"\"Standardize unicode forms and special Akkadian characters.\"\"\"\n    text = unicodedata.normalize(\"NFKC\", text)\n    \n    # Normalize H variants (competition note)\n    text = text.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n    \n    # Convert subscript numbers to normal numbers\n    subscript_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    text = text.translate(subscript_map)\n    \n    return text\n\n\n############################################\n# 2. REMOVE MODERN EDITORIAL MARKUP\n############################################\n\ndef remove_editorial_marks(text):\n    \"\"\"\n    Remove modern Assyriology notation that should not be learned by the model.\n    \"\"\"\n    # Remove certainty/uncertainty marks\n    text = re.sub(r\"[!?]\", \"\", text)\n    \n    # Remove line dividers and word dividers\n    text = re.sub(r\"[/:]\", \" \", text)\n\n    # Remove parentheses but keep content\n    text = re.sub(r\"[()]\", \"\", text)\n\n    # Remove partial sign brackets\n    text = re.sub(r\"[˹˺]\", \"\", text)\n\n    return text\n\n\n############################################\n# 3. HANDLE BROKEN TEXT & GAPS\n############################################\n\ndef normalize_gaps(text):\n    \"\"\"\n    Standardize gap notation into special tokens.\n    \"\"\"\n    # Large breaks\n    text = re.sub(r\"\\[\\s*…\\s*\\]\", \" <big_gap> \", text)\n    text = re.sub(r\"\\[….*?\\]\", \" <big_gap> \", text)\n\n    # Single broken sign\n    text = re.sub(r\"\\[x\\]\", \" <gap> \", text)\n    text = re.sub(r\"…\", \" <gap> \", text)\n\n    # Remove square brackets but keep contents\n    text = re.sub(r\"\\[([^\\]]+)\\]\", r\"\\1\", text)\n\n    return text\n\n\n############################################\n# 4. HANDLE SCRIBAL INSERTIONS\n############################################\n\ndef handle_insertions(text):\n    \"\"\"\n    Remove angle brackets but keep inserted text.\n    \"\"\"\n    text = re.sub(r\"[<>]\", \"\", text)\n    return text\n\n\n############################################\n# 5. DETERMINATIVE SEPARATION\n############################################\n\ndef separate_determinatives(text):\n    \"\"\"\n    Ensure determinatives in { } are separate tokens.\n    Example: a-lim{ki} → a-lim {ki}\n    \"\"\"\n    text = re.sub(r\"(\\w)\\{\", r\"\\1 {\", text)\n    text = re.sub(r\"\\}(\\w)\", r\"} \\1\", text)\n    return text\n\n\n############################################\n# 6. CLEAN EXTRA WHITESPACE\n############################################\n\ndef clean_whitespace(text):\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\n\n############################################\n# FULL PIPELINE\n############################################\n\ndef preprocess_akkadian(text):\n    text = normalize_unicode(text)\n    text = remove_editorial_marks(text)\n    text = normalize_gaps(text)\n    text = handle_insertions(text)\n    text = separate_determinatives(text)\n    text = clean_whitespace(text)\n    return text\n\n\n############################################\n# EXAMPLE\n############################################\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:43:54.409620Z","iopub.execute_input":"2026-02-01T10:43:54.409963Z","iopub.status.idle":"2026-02-01T10:43:54.422602Z","shell.execute_reply.started":"2026-02-01T10:43:54.409934Z","shell.execute_reply":"2026-02-01T10:43:54.421643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"akk_clean\"] = train[\"transliteration\"].apply(preprocess_akkadian)\ntest[\"akk_clean\"] = test[\"transliteration\"].apply(preprocess_akkadian)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:44:16.115060Z","iopub.execute_input":"2026-02-01T10:44:16.115463Z","iopub.status.idle":"2026-02-01T10:44:16.396718Z","shell.execute_reply.started":"2026-02-01T10:44:16.115431Z","shell.execute_reply":"2026-02-01T10:44:16.395707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:44:43.795058Z","iopub.execute_input":"2026-02-01T10:44:43.795476Z","iopub.status.idle":"2026-02-01T10:44:43.818040Z","shell.execute_reply.started":"2026-02-01T10:44:43.795444Z","shell.execute_reply":"2026-02-01T10:44:43.816966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:45:30.580551Z","iopub.execute_input":"2026-02-01T10:45:30.580910Z","iopub.status.idle":"2026-02-01T10:45:30.592989Z","shell.execute_reply.started":"2026-02-01T10:45:30.580882Z","shell.execute_reply":"2026-02-01T10:45:30.592022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_english(text):\n    text = str(text)\n    text = text.replace(\"  \", \" \")\n    text = text.strip()\n    return text\n\ntrain[\"en_clean\"] = train[\"translation\"].apply(preprocess_english)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:51:41.683135Z","iopub.execute_input":"2026-02-01T10:51:41.683575Z","iopub.status.idle":"2026-02-01T10:51:41.693387Z","shell.execute_reply.started":"2026-02-01T10:51:41.683544Z","shell.execute_reply":"2026-02-01T10:51:41.692163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_english(text):\n    text = str(text)\n    text = text.replace(\"  \", \" \")\n    text = text.strip()\n    return text\n\ntrain[\"en_clean\"] = train[\"translation\"].apply(preprocess_english)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:51:50.898143Z","iopub.execute_input":"2026-02-01T10:51:50.898490Z","iopub.status.idle":"2026-02-01T10:51:50.907531Z","shell.execute_reply.started":"2026-02-01T10:51:50.898461Z","shell.execute_reply":"2026-02-01T10:51:50.905941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:52:58.754570Z","iopub.execute_input":"2026-02-01T10:52:58.754936Z","iopub.status.idle":"2026-02-01T10:52:58.766317Z","shell.execute_reply.started":"2026-02-01T10:52:58.754907Z","shell.execute_reply":"2026-02-01T10:52:58.765433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lex = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\")\nlex.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:52:00.944971Z","iopub.execute_input":"2026-02-01T10:52:00.945330Z","iopub.status.idle":"2026-02-01T10:52:01.101655Z","shell.execute_reply.started":"2026-02-01T10:52:00.945302Z","shell.execute_reply":"2026-02-01T10:52:01.100653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"proper_nouns = set(lex[\"norm\"].dropna().str.lower())\n\ndef tag_names(text):\n    words = text.split()\n    tagged = []\n    for w in words:\n        if w.lower() in proper_nouns:\n            tagged.append(\"<NAME>\")\n        else:\n            tagged.append(w)\n    return \" \".join(tagged)\n\ntrain[\"akk_tagged\"] = train[\"akk_clean\"].apply(tag_names)\ntest[\"akk_tagged\"] = test[\"akk_clean\"].apply(tag_names)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:55:18.279627Z","iopub.execute_input":"2026-02-01T10:55:18.280029Z","iopub.status.idle":"2026-02-01T10:55:18.349812Z","shell.execute_reply.started":"2026-02-01T10:55:18.279996Z","shell.execute_reply":"2026-02-01T10:55:18.348647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T10:55:38.487444Z","iopub.execute_input":"2026-02-01T10:55:38.488188Z","iopub.status.idle":"2026-02-01T10:55:38.500043Z","shell.execute_reply.started":"2026-02-01T10:55:38.488151Z","shell.execute_reply":"2026-02-01T10:55:38.498821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_train = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\nfull_train[\"akk_clean\"] = full_train[\"transliteration\"].apply(preprocess_akkadian)\nfull_train[\"en_clean\"] = full_train[\"translation\"].apply(preprocess_english)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T11:08:16.333375Z","iopub.execute_input":"2026-02-01T11:08:16.333754Z","iopub.status.idle":"2026-02-01T11:08:16.759311Z","shell.execute_reply.started":"2026-02-01T11:08:16.333724Z","shell.execute_reply":"2026-02-01T11:08:16.756099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"akk_corpus.txt\", \"w\", encoding=\"utf-8\") as f:\n    for line in full_train[\"akk_clean\"]:\n        f.write(line + \"\\n\")\n\nwith open(\"en_corpus.txt\", \"w\", encoding=\"utf-8\") as f:\n    for line in full_train[\"en_clean\"]:\n        f.write(line + \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T11:08:18.261014Z","iopub.execute_input":"2026-02-01T11:08:18.262900Z","iopub.status.idle":"2026-02-01T11:08:18.281961Z","shell.execute_reply.started":"2026-02-01T11:08:18.262858Z","shell.execute_reply":"2026-02-01T11:08:18.279935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(full_train))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T11:08:19.494241Z","iopub.execute_input":"2026-02-01T11:08:19.494663Z","iopub.status.idle":"2026-02-01T11:08:19.500600Z","shell.execute_reply.started":"2026-02-01T11:08:19.494616Z","shell.execute_reply":"2026-02-01T11:08:19.499540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(full_train, test_size=0.1, random_state=42)\n\nprint(len(train_df), len(val_df))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-01T11:13:15.173781Z","iopub.execute_input":"2026-02-01T11:13:15.174171Z","iopub.status.idle":"2026-02-01T11:13:16.314995Z","shell.execute_reply.started":"2026-02-01T11:13:15.174144Z","shell.execute_reply":"2026-02-01T11:13:16.313737Z"}},"outputs":[],"execution_count":null}]}