{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Processing Training Data\n\nThere is little discussion on how processing training data should work. It is assumed that most teams are using proprietary methods based off of the information in the contest description, but I wanted to share a basic version that I found some success with in the hopes that it could be a launching point for others.\n\nHere, the majority of the code will look very familiar. However, the handling of gaps and large gaps, plus digit-periods, is enhanced beyond some notebooks I've seen. \n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:23:20.881291Z","iopub.execute_input":"2026-01-17T01:23:20.881649Z","iopub.status.idle":"2026-01-17T01:23:21.285632Z","shell.execute_reply.started":"2026-01-17T01:23:20.881607Z","shell.execute_reply":"2026-01-17T01:23:21.284464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filepath = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\ndf = pd.read_csv(filepath)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:23:21.287238Z","iopub.execute_input":"2026-01-17T01:23:21.287760Z","iopub.status.idle":"2026-01-17T01:23:21.351540Z","shell.execute_reply.started":"2026-01-17T01:23:21.287720Z","shell.execute_reply":"2026-01-17T01:23:21.350625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_transliteration(text):\n\n    # 1. Remove specific single punctuation and symbols: !, ?, /, :, — (em dash)\n    # Angle brackets < > are handled by the <.*?> pattern later.\n    # The period '.' is handled specifically to keep periods between numbers.\n    text = re.sub(r'[!?⁄:—]', '', text)\n\n    # Remove periods that are not between two digits.\n    # - Periods between numbers (e.g., '0.83333') will be kept.\n    # - Periods between letters (e.g., 'KÙ.BABBAR') will be removed.\n    # - Periods at the end/start of words or standalone will be removed.\n    text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text)\n    # 2. Remove patterns: < > (content inside), ˹ ˺ (content inside), [ ] (content inside)\n    # The non-greedy quantifier `*?` ensures it matches the smallest possible string.\n    text = re.sub(r'<.*?>', '', text)\n    text = re.sub(r'˹.*?˺', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text)\n\n    # 3. Replace subscripted numbers with their non-subscripted counterparts\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n    }\n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n\n    # Clean up any extra spaces that might result from removals (e.g., double spaces to single space)\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n\nprint(\"Text cleaning function 'clean_transliteration' defined.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:23:21.352744Z","iopub.execute_input":"2026-01-17T01:23:21.353143Z","iopub.status.idle":"2026-01-17T01:23:21.365437Z","shell.execute_reply.started":"2026-01-17T01:23:21.353103Z","shell.execute_reply":"2026-01-17T01:23:21.364467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'\\[…\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.\\.\\.\\]', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'(x)+', '<gap>', text)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:23:21.366722Z","iopub.execute_input":"2026-01-17T01:23:21.367086Z","iopub.status.idle":"2026-01-17T01:23:21.392204Z","shell.execute_reply.started":"2026-01-17T01:23:21.367045Z","shell.execute_reply":"2026-01-17T01:23:21.390888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_gaps(text):\n  if pd.isna(text): \n        return text\n  text = re.sub(r'<gap> x ', ' <gap> ', text)\n  text = re.sub(r'x <gap>', ' <gap>', text)\n  text = re.sub(r'<gap> <gap>', '<gap>', text)\n  text = re.sub(r'(<gap> )+', r'\\1', text)\n  return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:23:21.393647Z","iopub.execute_input":"2026-01-17T01:23:21.394079Z","iopub.status.idle":"2026-01-17T01:23:21.423671Z","shell.execute_reply.started":"2026-01-17T01:23:21.394038Z","shell.execute_reply":"2026-01-17T01:23:21.422406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_translation(text):\n    # Normalize all ellipsis types to a single token\n    text = text.replace('…', '...')\n\n    # Replace any bracketed gap like [...], [... ...], [....], [.. etc\n    text = re.sub(r'\\[\\s*\\.*\\s*(?:\\.*\\s*)*\\]', '<gap>', text)\n\n    # Replace any opening bracket followed by dots: [..., [.... etc\n    text = re.sub(r'\\[\\s*\\.+', '<gap>', text)\n\n    # Replace standalone ellipses (three or more dots) with <gap>\n    text = re.sub(r'\\.{3,}', '<gap>', text)\n\n    # Remove any remaining stray closing brackets\n    text = text.replace(']', '')\n\n    # Collapse multiple gaps into one\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n\n    # Clean up spacing\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:36:23.412326Z","iopub.execute_input":"2026-01-17T01:36:23.412692Z","iopub.status.idle":"2026-01-17T01:36:23.419886Z","shell.execute_reply.started":"2026-01-17T01:36:23.412662Z","shell.execute_reply":"2026-01-17T01:36:23.418658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['cleaned_akkadian'] = df['transliteration'].apply(clean_transliteration)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(replace_gaps)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(clean_gaps)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(clean_gaps) #Twice, because there are some weird double and triple gaps that take replacing.\nprint(\"Cleaned Akkadian column added to DataFrame.\")\n\n\ndf['cleaned_translation'] = df['translation'].apply(clean_translation)\n#The translations are rather clean besides gaps, it's just occasional brackets and such out of place. \n\n# Display a sample of original and cleaned text\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:36:26.177230Z","iopub.execute_input":"2026-01-17T01:36:26.177609Z","iopub.status.idle":"2026-01-17T01:36:27.038595Z","shell.execute_reply.started":"2026-01-17T01:36:26.177577Z","shell.execute_reply":"2026-01-17T01:36:27.037527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.to_csv('/kaggle/working/trainProcessed.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T01:36:30.467033Z","iopub.execute_input":"2026-01-17T01:36:30.467425Z","iopub.status.idle":"2026-01-17T01:36:30.571526Z","shell.execute_reply.started":"2026-01-17T01:36:30.467389Z","shell.execute_reply":"2026-01-17T01:36:30.569892Z"}},"outputs":[],"execution_count":null}]}