{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-17T06:56:05.432469Z","iopub.execute_input":"2026-01-17T06:56:05.432811Z","iopub.status.idle":"2026-01-17T06:56:05.441054Z","shell.execute_reply.started":"2026-01-17T06:56:05.432782Z","shell.execute_reply":"2026-01-17T06:56:05.440300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nTrain_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\ndf = pd.read_csv(Train_DATA_PATH)\ndf.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T06:56:05.442413Z","iopub.execute_input":"2026-01-17T06:56:05.442718Z","iopub.status.idle":"2026-01-17T06:56:05.491953Z","shell.execute_reply.started":"2026-01-17T06:56:05.442685Z","shell.execute_reply":"2026-01-17T06:56:05.491221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_transliteration(text):\n\n    # 1. Remove specific single punctuation and symbols: !, ?, /, :, — (em dash)\n    # Angle brackets < > are handled by the <.*?> pattern later.\n    # The period '.' is handled specifically to keep periods between numbers.\n    text = re.sub(r'[!?⁄:—]', '', text)\n\n    # Remove periods that are not between two digits.\n    # - Periods between numbers (e.g., '0.83333') will be kept.\n    # - Periods between letters (e.g., 'KÙ.BABBAR') will be removed.\n    # - Periods at the end/start of words or standalone will be removed.\n    text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text)\n    # 2. Remove patterns: < > (content inside), ˹ ˺ (content inside), [ ] (content inside)\n    # The non-greedy quantifier `*?` ensures it matches the smallest possible string.\n    text = re.sub(r'<.*?>', '', text)\n    text = re.sub(r'˹.*?˺', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text)\n\n    # 3. Replace subscripted numbers with their non-subscripted counterparts\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n    }\n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n\n    # Clean up any extra spaces that might result from removals (e.g., double spaces to single space)\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n\nprint(\"Text cleaning function 'clean_transliteration' defined.\")\n\ndef replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'\\[…\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.\\.\\.\\]', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'(x)+', '<gap>', text)\n\n    return text\n\ndef clean_gaps(text):\n  if pd.isna(text): \n        return text\n  text = re.sub(r'<gap> x ', ' <gap> ', text)\n  text = re.sub(r'x <gap>', ' <gap>', text)\n  text = re.sub(r'<gap> <gap>', '<gap>', text)\n  text = re.sub(r'(<gap> )+', r'\\1', text)\n  return text\n\ndef clean_translation(text):\n    # Normalize all ellipsis types to a single token\n    text = text.replace('…', '...')\n\n    # Replace any bracketed gap like [...], [... ...], [....], [.. etc\n    text = re.sub(r'\\[\\s*\\.*\\s*(?:\\.*\\s*)*\\]', '<gap>', text)\n\n    # Replace any opening bracket followed by dots: [..., [.... etc\n    text = re.sub(r'\\[\\s*\\.+', '<gap>', text)\n\n    # Replace standalone ellipses (three or more dots) with <gap>\n    text = re.sub(r'\\.{3,}', '<gap>', text)\n\n    # Remove any remaining stray closing brackets\n    text = text.replace(']', '')\n\n    # Collapse multiple gaps into one\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n\n    # Clean up spacing\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n\ndf['cleaned_akkadian'] = df['transliteration'].apply(clean_transliteration)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(replace_gaps)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(clean_gaps)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(clean_gaps) #Twice, because there are some weird double and triple gaps that take replacing.\nprint(\"Cleaned Akkadian column added to DataFrame.\")\n\n\ndf['cleaned_translation'] = df['translation'].apply(clean_translation)\n\n# Display a sample of original and cleaned text\ndf.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T06:56:05.493755Z","iopub.execute_input":"2026-01-17T06:56:05.494190Z","iopub.status.idle":"2026-01-17T06:56:06.284695Z","shell.execute_reply.started":"2026-01-17T06:56:05.494136Z","shell.execute_reply":"2026-01-17T06:56:06.283970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.to_csv(\"submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T07:01:45.270349Z","iopub.execute_input":"2026-01-17T07:01:45.270674Z","iopub.status.idle":"2026-01-17T07:01:45.361905Z","shell.execute_reply.started":"2026-01-17T07:01:45.270643Z","shell.execute_reply":"2026-01-17T07:01:45.361312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}