{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Processing Training Data\n\nref: https://www.kaggle.com/code/teruzuki061/akkadianfurtherdataprocessing","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:33:44.923307Z","iopub.execute_input":"2025-12-31T07:33:44.923580Z","iopub.status.idle":"2025-12-31T07:33:46.341146Z","shell.execute_reply.started":"2025-12-31T07:33:44.923553Z","shell.execute_reply":"2025-12-31T07:33:46.340105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filepath = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\ndf = pd.read_csv(filepath)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:33:46.343252Z","iopub.execute_input":"2025-12-31T07:33:46.343736Z","iopub.status.idle":"2025-12-31T07:33:46.445358Z","shell.execute_reply.started":"2025-12-31T07:33:46.343704Z","shell.execute_reply":"2025-12-31T07:33:46.444299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nclass DataCleaner():\n    def __init__(self, df, inplace=True):\n        self.df = df\n        self.inplace = inplace\n\n    def _replace_gaps(self, text):\n        if pd.isna(text): \n            return text\n        \n        text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n        text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]', '<big_gap>', text)\n        text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    \n        text = re.sub(r'\\[x\\]', '<gap>', text)\n        text = re.sub(r' x ', ' <gap> ', text)\n        text = re.sub(r'\\[…\\]', '<big_gap>', text)\n        text = re.sub(r'\\[\\.\\.\\.\\]', '<big_gap>', text)\n        text = re.sub(r'…', '<big_gap>', text)\n        text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n        text = re.sub(r'(x)+', '<gap>', text)\n    \n        return text\n\n    def _clean_gaps(self, text):\n      if pd.isna(text): \n            return text\n      text = re.sub(r'<gap> x ', ' <gap> ', text)\n      text = re.sub(r'x <gap>', ' <gap>', text)\n      text = re.sub(r'<gap> <gap>', '<gap>', text)\n      text = re.sub(r'(<gap> )+', r'\\1', text)\n      return text\n\n    def _clean_transliteration(self, text):\n    \n        # 1. Remove specific single punctuation and symbols: !, ?, /, :, — (em dash)\n        # Angle brackets < > are handled by the <.*?> pattern later.\n        # The period '.' is handled specifically to keep periods between numbers.\n        text = re.sub(r'[!?⁄:—]', '', text)\n    \n        # Remove periods that are not between two digits.\n        # This handles the user's request:\n        # - Periods between numbers (e.g., '0.83333') will be kept.\n        # - Periods between letters (e.g., 'KÙ.BABBAR') will be removed.\n        # - Periods at the end/start of words or standalone will be removed.\n        text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text)\n        # 2. Remove patterns: < > (content inside), ˹ ˺ (content inside), [ ] (content inside)\n        # The non-greedy quantifier `*?` ensures it matches the smallest possible string.\n        text = re.sub(r'<.*?>', '', text)\n        text = re.sub(r'˹.*?˺', '', text)\n        text = re.sub(r'\\[.*?\\]', '', text)\n    \n        # 3. Replace subscripted numbers with their non-subscripted counterparts\n        subscript_map = {\n            '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n            '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n        }\n        for sub, normal in subscript_map.items():\n            text = text.replace(sub, normal)\n    \n        # Clean up any extra spaces that might result from removals (e.g., double spaces to single space)\n        text = re.sub(r'\\s+', ' ', text).strip()\n    \n        return text\n\n        \n    def clean_transliterations(self, colnames: list, df=None, inplace: bool=None):\n        inplace = self.inplace if inplace is None else inplace\n        df = self.df if df is None else df\n        df = df.copy() if not inplace else df\n        # check\n        for colname in colnames:\n            if not(colname in df.columns):\n                raise valueError(\n                    f\"Column name \\'{colname}\\' not found in df.columns.\\n\"\\\n                    f\"df.columns : {df.columns}\"\n                )\n        # apply\n        for colname in colnames:\n            df[colname] = df[colname]\\\n                .apply(self._clean_transliteration)\\\n                .apply(self._replace_gaps)\\\n                .apply(self._clean_gaps)\n\n        return df\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:33:55.092696Z","iopub.execute_input":"2025-12-31T07:33:55.093088Z","iopub.status.idle":"2025-12-31T07:33:55.107983Z","shell.execute_reply.started":"2025-12-31T07:33:55.093054Z","shell.execute_reply":"2025-12-31T07:33:55.106918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_cleaner = DataCleaner(df=df)\ndf = data_cleaner.clean_transliterations(['transliteration'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:33:56.832264Z","iopub.execute_input":"2025-12-31T07:33:56.832731Z","iopub.status.idle":"2025-12-31T07:33:57.042607Z","shell.execute_reply.started":"2025-12-31T07:33:56.832687Z","shell.execute_reply":"2025-12-31T07:33:57.041435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Cleaned Akkadian column added to DataFrame.\")\n\n# Display a sample of original and cleaned text\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:33:57.853643Z","iopub.execute_input":"2025-12-31T07:33:57.853982Z","iopub.status.idle":"2025-12-31T07:33:57.864253Z","shell.execute_reply.started":"2025-12-31T07:33:57.853953Z","shell.execute_reply":"2025-12-31T07:33:57.863274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df.to_csv(\"clean_train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T07:34:24.867170Z","iopub.execute_input":"2025-12-31T07:34:24.867616Z","iopub.status.idle":"2025-12-31T07:34:24.940304Z","shell.execute_reply.started":"2025-12-31T07:34:24.867580Z","shell.execute_reply":"2025-12-31T07:34:24.939094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}