{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport math\nfrom collections import Counter\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:07.958293Z","iopub.execute_input":"2026-01-13T10:19:07.958642Z","iopub.status.idle":"2026-01-13T10:19:08.309226Z","shell.execute_reply.started":"2026-01-13T10:19:07.958611Z","shell.execute_reply":"2026-01-13T10:19:08.308233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\nsentences_oare = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\")\noa_lexicon_ebl = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\")\npublications = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/publications.csv\")\nebl_dictionary = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:08.310950Z","iopub.execute_input":"2026-01-13T10:19:08.311791Z","iopub.status.idle":"2026-01-13T10:19:17.941169Z","shell.execute_reply.started":"2026-01-13T10:19:08.311750Z","shell.execute_reply":"2026-01-13T10:19:17.940382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# `train.csv` - the main source of transliteration-translation pairs","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:17.942211Z","iopub.execute_input":"2026-01-13T10:19:17.942534Z","iopub.status.idle":"2026-01-13T10:19:17.967782Z","shell.execute_reply.started":"2026-01-13T10:19:17.942501Z","shell.execute_reply":"2026-01-13T10:19:17.966745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:17.970014Z","iopub.execute_input":"2026-01-13T10:19:17.970275Z","iopub.status.idle":"2026-01-13T10:19:17.992488Z","shell.execute_reply.started":"2026-01-13T10:19:17.970252Z","shell.execute_reply":"2026-01-13T10:19:17.991325Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Based on the number of unique entries some transliterations and translations seem duplicated.","metadata":{}},{"cell_type":"markdown","source":"# Remove duplicated entries","metadata":{}},{"cell_type":"code","source":"# Helper statistics for differentiation of strings\ntrain['transliteration_charlen']=train['transliteration'].str.len()\ntrain['translation_charlen']=train['translation'].str.len()\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:17.993533Z","iopub.execute_input":"2026-01-13T10:19:17.993929Z","iopub.status.idle":"2026-01-13T10:19:18.006886Z","shell.execute_reply.started":"2026-01-13T10:19:17.993898Z","shell.execute_reply":"2026-01-13T10:19:18.005869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Duplicated entries (transliterations AND translations)","metadata":{}},{"cell_type":"code","source":"train[train.duplicated(subset=['transliteration', 'translation'], keep=False)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.007959Z","iopub.execute_input":"2026-01-13T10:19:18.008236Z","iopub.status.idle":"2026-01-13T10:19:18.038090Z","shell.execute_reply.started":"2026-01-13T10:19:18.008202Z","shell.execute_reply":"2026-01-13T10:19:18.037250Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are no entries with identical transliteration-translations pairs, which means there are duplicated transliterations with different translations and vice versa.","metadata":{}},{"cell_type":"markdown","source":"## Duplicated transliterations with same length translations","metadata":{}},{"cell_type":"code","source":"train[train.duplicated(subset='transliteration', keep=False)].sort_values(by=['transliteration'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.039112Z","iopub.execute_input":"2026-01-13T10:19:18.039519Z","iopub.status.idle":"2026-01-13T10:19:18.050337Z","shell.execute_reply.started":"2026-01-13T10:19:18.039492Z","shell.execute_reply":"2026-01-13T10:19:18.049382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Some transliterations appear the same but have longer accompanying translations. We will keep those.","metadata":{}},{"cell_type":"code","source":"train=train.sort_values(by=['transliteration_charlen', 'translation_charlen'], ascending=[True, False]).drop_duplicates(subset='transliteration', keep='first')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.051464Z","iopub.execute_input":"2026-01-13T10:19:18.051784Z","iopub.status.idle":"2026-01-13T10:19:18.073678Z","shell.execute_reply.started":"2026-01-13T10:19:18.051748Z","shell.execute_reply":"2026-01-13T10:19:18.072793Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Duplicated translations with same length transliterations","metadata":{}},{"cell_type":"code","source":"train[train.duplicated(subset='translation', keep=False)].sort_values(by=['translation'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.074878Z","iopub.execute_input":"2026-01-13T10:19:18.075182Z","iopub.status.idle":"2026-01-13T10:19:18.087179Z","shell.execute_reply.started":"2026-01-13T10:19:18.075148Z","shell.execute_reply":"2026-01-13T10:19:18.086279Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Similar to the transliterations case, some translations have the same length but longer corresponding transliterations. We save the ones with the longer corresponding transliterations.","metadata":{}},{"cell_type":"code","source":"train=train.sort_values(by=['transliteration_charlen', 'translation_charlen'], ascending=[False, True]).drop_duplicates(subset='translation', keep='first')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.090194Z","iopub.execute_input":"2026-01-13T10:19:18.090901Z","iopub.status.idle":"2026-01-13T10:19:18.105949Z","shell.execute_reply.started":"2026-01-13T10:19:18.090867Z","shell.execute_reply":"2026-01-13T10:19:18.105025Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Character analysis","metadata":{}},{"cell_type":"code","source":"transliteration_megatext = (train['transliteration']+' ').sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.107085Z","iopub.execute_input":"2026-01-13T10:19:18.107427Z","iopub.status.idle":"2026-01-13T10:19:18.230845Z","shell.execute_reply.started":"2026-01-13T10:19:18.107393Z","shell.execute_reply":"2026-01-13T10:19:18.229875Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Unique characters in the transliterations - case sensitive","metadata":{}},{"cell_type":"code","source":"transliteration_unique = ''.join(sorted(list(set(transliteration_megatext))))\ntransliteration_unique","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.232384Z","iopub.execute_input":"2026-01-13T10:19:18.232771Z","iopub.status.idle":"2026-01-13T10:19:18.262170Z","shell.execute_reply.started":"2026-01-13T10:19:18.232721Z","shell.execute_reply":"2026-01-13T10:19:18.261122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Unique characters in the transliteration - case insensitive","metadata":{}},{"cell_type":"code","source":"transliteration_unique_ci = ''.join(sorted(list(set(transliteration_megatext.lower()))))\ntransliteration_unique_ci","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.263408Z","iopub.execute_input":"2026-01-13T10:19:18.263789Z","iopub.status.idle":"2026-01-13T10:19:18.296333Z","shell.execute_reply.started":"2026-01-13T10:19:18.263750Z","shell.execute_reply":"2026-01-13T10:19:18.295541Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to the [Overview: Additional Characters & Formats](https://www.kaggle.com/competitions/deep-past-initiative-machine-translation/overview) the acceptable characters are:\n\n| Character | CDLI | ORACC | Unicode |\n| --- | --- | --- | --- |\n| á | a2 | a₂ | \n| à | a3 | a₃ | \n| é | e2 | e₂ | \n| è | e3 | e₃ | \n| í | i2 | i₂ | \n| ì | i3 | i₃ | \n| ú | u2 | u₂ | \n| ù | u3 | u₃ | \n| š | sz | š | U+161 |\n| Š | SZ | Š | U+160 |\n| Ṣ | s, | ṣ | U+1E63 |\n| ṣ | S, | Ṣ | U+1E62 |\n| ṭ | t, | ṭ | U+1E6D |\n| Ṭ | T, | Ṭ | U+1E6C |\n| ‘ | ‘ | ʾ | U+02BE |\n| 0-9 | 0-9 | subscript ₀-₉ | U+2080-U+2089 |\n| xₓ | Xx | subscript ₓ | U+208A |\n| ḫ | h | h | U+1E2B |\n| Ḫ | H | H | U+1E2A |\n\nFrom this we can relate see that the characters mostly follow the ORACC encoding with the exception of accents 1-3, e.g., (a₁=a, a₂=á, a₃=à)\n\n```python\ndef oracc2dpc(text):\n    swaps=[\n        ['grave_acute', zip(['a₂','a₃','e₂','e₃','i₂','i₃','u₂','u₃'],\n                            ['à','á','è','é','ì','í','ù','ú'])],\n        ['subscripts', zip('₀₁₂₃₄₅₆₇₈₉', '0123456789')],\n        ['other', zip('ʾhH', '‘ḫḪ')]\n    ]\n    for char_group in swaps:\n        name, swap_pairs = char_group\n        for _before, _after in swap_pairs:\n            if isinstance(text, str):\n                text = text.replace(_before, _after)\n            elif isinstance(text, pd.Series):\n                text = text.str.replace(_before, _after)\n            else:\n                raise Exception(\"Wrong text format.\")\n    return text\n```\n\nThe function is not used here but may be used in future for deeper analysis of texts other than those in `train.csv`","metadata":{}},{"cell_type":"code","source":"def oracc2dpc(text):\n    swaps=[\n        ['grave_acute', zip(['a₂','a₃','e₂','e₃','i₂','i₃','u₂','u₃'],\n                            ['à','á','è','é','ì','í','ù','ú'])],\n        ['subscripts', zip('₀₁₂₃₄₅₆₇₈₉', '0123456789')],\n        ['other', zip('ʾhH', '‘ḫḪ')]\n    ]\n    for char_group in swaps:\n        name, swap_pairs = char_group\n        for _before, _after in swap_pairs:\n            if isinstance(text, str):\n                text = text.replace(_before, _after)\n            elif isinstance(text, pd.Series):\n                text = text.str.replace(_before, _after)\n            else:\n                raise Exception(\"Wrong text format.\")\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T16:53:20.581066Z","iopub.execute_input":"2026-01-14T16:53:20.581528Z","iopub.status.idle":"2026-01-14T16:53:20.593728Z","shell.execute_reply.started":"2026-01-14T16:53:20.581487Z","shell.execute_reply":"2026-01-14T16:53:20.592618Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## A look into the surrounding of the selected characters","metadata":{}},{"cell_type":"code","source":"def get_texts_w_pat(pat=r'break', _from='transliteration', groupbyid=True):\n    esc_pat=f'[{pat}]'\n    total_unique_texts=train.loc[train[_from].str.contains(esc_pat, regex=True), ['oare_id']].nunique().iloc[0]\n    # tot_char_count=transliteration_char_count.sum()\n\n    if groupbyid:\n        filtered_data = pd.concat(\n            [\n                train.loc[train[_from].str.contains(esc_pat, regex=True), ['oare_id', _from]],\n                train[_from].str.extractall(r'(\\b.{,10}'+esc_pat+'.{,10})').reset_index().rename(columns={\n                                                                                                'level_0': 'index',\n                                                                                                0: 'matches'}\n                                                                                            ).groupby('index')['matches'].apply(list)\n            ], axis=1\n        )\n        \n        print(f\"= = = TOTAL UNIQUE TEXTS WITH PATTERN ({pat}): {total_unique_texts} = = =\")\n        for oare_id in filtered_data['oare_id'].unique():\n            print(f\"=== OARE_ID: {oare_id} ({len(filtered_data[filtered_data['oare_id']==oare_id]['matches'].values[0])} matches)===\")\n            print(f\"\\tmatches: {'  ,  '.join(filtered_data[filtered_data['oare_id']==oare_id]['matches'].values[0])}\")\n            print(\"\\n\")\n    else:\n        matches = train[_from].str.extractall(r'(\\b.{,10}'+esc_pat+'.{,10})').reset_index().rename(columns={\n                                                                                                'level_0': 'index',\n                                                                                                0: 'matches'}\n                                                                                            )#.groupby('index')['matches'].apply(list)\n        print(f\"= = = TOTAL UNIQUE TEXTS WITH PATTERN (  '{pat}'  ): {int(total_unique_texts)}. TOTAL MATCHES: {matches['matches'].count()} = = =\")\n        print(f\"\\tmatches: {'  ,  '.join(matches['matches'].to_list()[:150])+'...'}\")\n        # for match in matches.values:\n        #     print(f\"=== Total matches: ({len(filtered_data[filtered_data['oare_id']==oare_id]['matches'].values[0])} matches)===\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.312922Z","iopub.execute_input":"2026-01-13T10:19:18.313764Z","iopub.status.idle":"2026-01-13T10:19:18.333703Z","shell.execute_reply.started":"2026-01-13T10:19:18.313695Z","shell.execute_reply":"2026-01-13T10:19:18.332819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for rare_sign in '()-ʾ…⌈[]+.':#transliteration_char_count_case_sensitive[transliteration_char_count_case_sensitive<tot_char_count*0.005].index:\n    print(''.join(['=' for _ in range(20)]))\n    get_texts_w_pat(pat=rare_sign, _from='transliteration', groupbyid=False)\n    print(\"\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.334695Z","iopub.execute_input":"2026-01-13T10:19:18.334986Z","iopub.status.idle":"2026-01-13T10:19:18.838222Z","shell.execute_reply.started":"2026-01-13T10:19:18.334960Z","shell.execute_reply":"2026-01-13T10:19:18.837199Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Usage of characters: `()-ʾ…⌈[]+.`\n\n### Round brackets - `()`\nUsed to denote:\n- determinatives, e.g., (d) - deity `(d)IM.ILLAT`, (TÚG) - textile `(TÚG)ku-ta-ni`, (ki) - earth, ground, place `a-lim(ki)`\n- large breaks and broken lines, e.g., (large break) - `(large break)`, (N broken lines) - `(3 broken lines)`\n\n### Hyphen - `-`\n- connections inside a transliterated word, e.g., `a-šùr-na-da`, `su-ku-ku`\n\n### Glottal stop - `ʾ`\nDenotes a glottal stop... In the `train.csv` used only once, but legitly in `ta-ʾi-tim` - information/instruction\n\n### Ellipsis - `…`\nDenotes a large break in the text.\n\n### Left ceiling symbol - `⌈`\nErroneous usage in `be-et⌈a-bi-ku-nu` = `be-et a-bi-ku-nu` - your father's house\n\n### Square brackets - `[]`\nUsed to denote large breaks in text with three dots `[...]`\n\n### Plus sign - `+`\nUsed to denote:\n- mathematical operation, e.g., `x-1+0.25 GÍN` - 1 1/4 shekels, `a-10+0.5 GÍN`\n- doubling of a sign, `train.csv` shows only a single example of `-lá+lá-`\n\n### Dot - `.`\nUsed to denote:\n- floating point separation, e.g., `1.8333300000`, `2.5`, `0.25`, `0.66666`\n- sign separation in compound words, e.g., `KÙ.BABBAR` - silver, `šu-(d)EN.LÍL` - Šu-Illil (personal pame), `ITU.1.KAM` - first month\n","metadata":{}},{"cell_type":"markdown","source":"## Syllables and signs","metadata":{}},{"cell_type":"code","source":"_signs_and_syllables = transliteration_megatext.replace(\"(\", \" (\").replace(\")\", \") \")\n_signs_and_syllables = _signs_and_syllables.replace(\".\", \" \").replace('-', \" \").replace(\"+\", \" \")\nsigns_and_syllables = pd.Series(Counter(_signs_and_syllables.split(\" \")),\n                                name='signs_and_syllables'\n                               ).sort_values(ascending=False)\nsigns_and_syllables = signs_and_syllables[[idx for idx in signs_and_syllables.index \\\n                                           if set(idx).intersection(set('0123456789'))==set()]]\nsigns_and_syllables","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.839408Z","iopub.execute_input":"2026-01-13T10:19:18.839778Z","iopub.status.idle":"2026-01-13T10:19:18.899962Z","shell.execute_reply.started":"2026-01-13T10:19:18.839741Z","shell.execute_reply":"2026-01-13T10:19:18.898907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"signs_and_syllables.head(30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.901180Z","iopub.execute_input":"2026-01-13T10:19:18.901520Z","iopub.status.idle":"2026-01-13T10:19:18.909350Z","shell.execute_reply.started":"2026-01-13T10:19:18.901479Z","shell.execute_reply":"2026-01-13T10:19:18.908399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"signs_and_syllables.index.nunique(), signs_and_syllables.index.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.910516Z","iopub.execute_input":"2026-01-13T10:19:18.910899Z","iopub.status.idle":"2026-01-13T10:19:18.924997Z","shell.execute_reply.started":"2026-01-13T10:19:18.910816Z","shell.execute_reply":"2026-01-13T10:19:18.924048Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The most popular words are set in the trade context: `ma-na` (mina, a unit of weight), `GÍN` (shekels, a unit of weight) `KÙ.BABBAR` (silver), `DUMU` (son of), `šur` (part of name A-šur/Aššur). The single `x` is interesting, since it shows how often the scholars deemed the sign unreadable.","metadata":{}},{"cell_type":"markdown","source":"## Comparison with the OA Lexicone\nIt is worth comparing the sign frequency counts as taken from the train set to the sign frequencies ","metadata":{}},{"cell_type":"markdown","source":"### A necessary cleanup - broken entries\nBefore anything the data, especially the `form` and `lexeme` columns should be cleaned up of any weird entries. So far 2 types have been found:\n- NaN in `form` column\n- 36 digit numbers instead of transliterations","metadata":{}},{"cell_type":"code","source":"oa_lexicon_ebl[oa_lexicon_ebl['form'].isna()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.926052Z","iopub.execute_input":"2026-01-13T10:19:18.926361Z","iopub.status.idle":"2026-01-13T10:19:18.948565Z","shell.execute_reply.started":"2026-01-13T10:19:18.926334Z","shell.execute_reply":"2026-01-13T10:19:18.947533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oa_lexicon_ebl.dropna(subset='form')[oa_lexicon_ebl.dropna(subset='form').loc[:, 'form'].str.isdigit()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:18.949735Z","iopub.execute_input":"2026-01-13T10:19:18.950005Z","iopub.status.idle":"2026-01-13T10:19:19.000370Z","shell.execute_reply.started":"2026-01-13T10:19:18.949972Z","shell.execute_reply":"2026-01-13T10:19:18.999542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Calculate the sign/syllable frequencies\nThe transliterations in the `form` section are cleaned and single space-split. Additionally, the signs are grouped by the lexeme and only unique ones are taken for further analysis -- some lexemes can have several `form` variants, so this is to prevent unnatural bias of words with many possible forms.","metadata":{}},{"cell_type":"code","source":"_unique_signs_oa_lexicone = oa_lexicon_ebl[['form', 'lexeme']].copy().dropna(subset='form')\n_unique_signs_oa_lexicone = _unique_signs_oa_lexicone[~_unique_signs_oa_lexicone['form'].str.isdigit()]\n_unique_signs_oa_lexicone['form'] += ' '\n_unique_signs_oa_lexicone.loc[:, 'form'] = _unique_signs_oa_lexicone['form'].str.replace('-', ' ')\\\n                                                                            .str.replace('.', ' ')\\\n                                                                            .str.replace('(', ' (')\\\n                                                                            .str.replace(')', ') ')\\\n                                                                            .str.replace('{', ' {')\\\n                                                                            .str.replace('}', '} ')\\\n                                                                            .str.replace('=', ' ')\n_unique_signs_oa_lexicone = _unique_signs_oa_lexicone.groupby('lexeme').sum()['form'].str.split(' ').apply(lambda x: ' '.join([str(i) for i in np.unique(x)]))\nunique_signs_oa_lexicone = pd.Series(Counter(_unique_signs_oa_lexicone.sum().split(' ')), name='unique_signs_oa_lexicone').sort_values(ascending=False)\nunique_signs_oa_lexicone.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.001418Z","iopub.execute_input":"2026-01-13T10:19:19.001859Z","iopub.status.idle":"2026-01-13T10:19:19.307196Z","shell.execute_reply.started":"2026-01-13T10:19:19.001800Z","shell.execute_reply":"2026-01-13T10:19:19.306443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_signs_oa_lexicone.index.nunique(), unique_signs_oa_lexicone.index.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.308325Z","iopub.execute_input":"2026-01-13T10:19:19.308668Z","iopub.status.idle":"2026-01-13T10:19:19.314663Z","shell.execute_reply.started":"2026-01-13T10:19:19.308634Z","shell.execute_reply":"2026-01-13T10:19:19.313870Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Signs/syllables found in `train.csv` and the OA Lexicon","metadata":{}},{"cell_type":"code","source":"shared_signs = sorted(list(set(unique_signs_oa_lexicone.index) & set(signs_and_syllables.index)))\nlen(shared_signs), np.array(shared_signs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.315686Z","iopub.execute_input":"2026-01-13T10:19:19.315946Z","iopub.status.idle":"2026-01-13T10:19:19.334226Z","shell.execute_reply.started":"2026-01-13T10:19:19.315923Z","shell.execute_reply":"2026-01-13T10:19:19.333299Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Signs/syllables absent in the `train.csv` but present in the OA Lexicone\nWe filter out the digit-like entries, coming from forms like `(ITU).1.KAM`","metadata":{}},{"cell_type":"code","source":"sign_unique_to_oa_lexicone = unique_signs_oa_lexicone.index[~unique_signs_oa_lexicone.index.isin(signs_and_syllables.index) & (~unique_signs_oa_lexicone.index.str.isdigit())].sort_values(ascending=True)\nsign_unique_to_oa_lexicone.shape, sign_unique_to_oa_lexicone.to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.335310Z","iopub.execute_input":"2026-01-13T10:19:19.335838Z","iopub.status.idle":"2026-01-13T10:19:19.354977Z","shell.execute_reply.started":"2026-01-13T10:19:19.335759Z","shell.execute_reply":"2026-01-13T10:19:19.354081Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Checking for signs unique for `train.csv` and NOT present in the OA Lexicone show fragments to clean up\nMost of the cases include:\n- several types of `<gap>`/`<big_gap>` notation, e.g., `x, xx, ..., [...], (N broken lines), (large break)`\n- single consonants, which are often found in the abruptly ended/cut transliterations ([32 such texts have been identified based on a cross-over analysis with Sentences_Oare_FirstWord_LinNum.csv](https://www.kaggle.com/code/angantyr/deep-past-2025-1st-word-in-sentence-idx-analysis#Problematic-texts))","metadata":{}},{"cell_type":"code","source":"signs_and_syllables.index[~signs_and_syllables.index.isin(unique_signs_oa_lexicone.index)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.356134Z","iopub.execute_input":"2026-01-13T10:19:19.356498Z","iopub.status.idle":"2026-01-13T10:19:19.370083Z","shell.execute_reply.started":"2026-01-13T10:19:19.356440Z","shell.execute_reply":"2026-01-13T10:19:19.369196Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Possibly broken forms with stray fragments `(m, (, )`\nThe list of unique signs shows three cases for possibly broken forms with stray round brackets: `(m, (, )`","metadata":{}},{"cell_type":"code","source":"oa_lexicon_ebl.dropna(subset='form')[oa_lexicon_ebl.dropna(subset='form')['form'].str.contains('(m', regex=False) \\\n                                     & ~oa_lexicon_ebl.dropna(subset='form')['form'].str.contains('(m)', regex=False)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.371213Z","iopub.execute_input":"2026-01-13T10:19:19.371532Z","iopub.status.idle":"2026-01-13T10:19:19.441786Z","shell.execute_reply.started":"2026-01-13T10:19:19.371481Z","shell.execute_reply":"2026-01-13T10:19:19.440995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Amurru as a personal name (PN) likely [refers to the Amorites](https://www.ebl.lmu.de/dictionary/amurru%20I), in which case the transliterated form would be `MAR.TU`\n\nUmu as a word has several meanings. The one I was considering is the word for day: `umum`. The [dictionary entry](https://www.ebl.lmu.de/dictionary/%C5%ABmu%20I) would give then a possible reading of `UD`, `ME` (astrological day) appearing in forms like `UD.X.KAM` (Xth day/X days) for which there is an entry in the dictionary.","metadata":{}},{"cell_type":"code","source":"oa_lexicon_ebl.dropna(subset='form')[oa_lexicon_ebl.dropna(subset='form')['form'].str.contains('UD\\.\\d+\\.KAM', regex=True)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.445593Z","iopub.execute_input":"2026-01-13T10:19:19.445897Z","iopub.status.idle":"2026-01-13T10:19:19.494403Z","shell.execute_reply.started":"2026-01-13T10:19:19.445870Z","shell.execute_reply":"2026-01-13T10:19:19.493397Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The stray round brackets `( and )` are false positives from the string replacement process as all come from the `(TÚG)(.)(ḪI)(.)(A)`-related forms.","metadata":{}},{"cell_type":"code","source":"oa_lexicon_ebl.dropna(subset='form')[oa_lexicon_ebl.dropna(subset='form')['form'].str.contains('\\(\\.\\)', regex=True)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.495474Z","iopub.execute_input":"2026-01-13T10:19:19.495811Z","iopub.status.idle":"2026-01-13T10:19:19.544809Z","shell.execute_reply.started":"2026-01-13T10:19:19.495776Z","shell.execute_reply":"2026-01-13T10:19:19.543958Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"All of the above refer to the specific types of textiles in their plural form, e.g., `(TÚG)(.)(ḪI)(.)(A)a-bar-ni-e` = abarniu textiles. This can be easily verified by finding all forms for a single piece of textile, i.e., beginning with `(TÚG)` determinative. This in fact gives much more examples that could help expand the plural forms present in the dictionary.","metadata":{}},{"cell_type":"code","source":"oa_lexicon_ebl.dropna(subset='form')[oa_lexicon_ebl.dropna(subset='form')['form'].str.contains('\\(TÚG\\)', regex=True)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.545847Z","iopub.execute_input":"2026-01-13T10:19:19.546231Z","iopub.status.idle":"2026-01-13T10:19:19.594839Z","shell.execute_reply.started":"2026-01-13T10:19:19.546204Z","shell.execute_reply":"2026-01-13T10:19:19.593949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Number representation in `train.csv`","metadata":{}},{"cell_type":"code","source":"train.loc[train['translation'].str.contains('talent', regex=False), 'transliteration'].iloc[0], train.loc[train['translation'].str.contains('talent', regex=False), 'translation'].iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.595989Z","iopub.execute_input":"2026-01-13T10:19:19.596364Z","iopub.status.idle":"2026-01-13T10:19:19.610064Z","shell.execute_reply.started":"2026-01-13T10:19:19.596336Z","shell.execute_reply":"2026-01-13T10:19:19.609113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transliteration_wgt_measures=train['transliteration'].str.extractall('((?:\\d+\\.*\\d*)*\\s*(?:(?:ma-na)|GÍN|GÚ|(?:me-at))(?:(?:\\s|\\sù\\s)(?:\\d+\\.*\\d*)\\s(?:(?:ma-na)|GÍN|GÚ))*)')\ntransliteration_wgt_measures.rename(columns={0: 'transliteration_wgt_measures'}, inplace=True)\ntransliteration_wgt_measures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.611201Z","iopub.execute_input":"2026-01-13T10:19:19.611519Z","iopub.status.idle":"2026-01-13T10:19:19.723649Z","shell.execute_reply.started":"2026-01-13T10:19:19.611467Z","shell.execute_reply":"2026-01-13T10:19:19.722892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"translation_wgt_measures=train['translation'].str.extractall('((?:(?:\\d+\\.*\\d*)|(?:(?:\\d+\\s?\\d+)\\s?/\\s?\\d+)|one|two|three|four|five|six|seven|eight|nine|ten)*\\s*(?:(?:mina[s]?)|s[hk]{1}e[hk]{1}el[s]?|talent[s]?))(?:(?:\\s|\\sand\\s|\\sor\\s)(?:(?:\\d+\\.*\\d*)|one|two|three|four|five|six|seven|eight|nine|ten)\\s(?:(?:mina[s]?)|s[hk]{1}e[hk]{1}el[s]?|talent[s]?))*')\ntranslation_wgt_measures.rename(columns={0: 'translation_wgt_measures'}, inplace=True)\ntranslation_wgt_measures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.725017Z","iopub.execute_input":"2026-01-13T10:19:19.725364Z","iopub.status.idle":"2026-01-13T10:19:19.901699Z","shell.execute_reply.started":"2026-01-13T10:19:19.725338Z","shell.execute_reply":"2026-01-13T10:19:19.900767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures=pd.concat(\n    [\n        transliteration_wgt_measures,\n        translation_wgt_measures\n    ], axis=1\n)#.sort_values(by='match', ascending=True)\nwgt_measures.index.set_names(['row_idx', 'match_num'], inplace=True)\nwgt_measures = wgt_measures.sort_values(by=['row_idx', 'match_num'], ascending=True)\nwgt_measures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.902926Z","iopub.execute_input":"2026-01-13T10:19:19.903251Z","iopub.status.idle":"2026-01-13T10:19:19.925640Z","shell.execute_reply.started":"2026-01-13T10:19:19.903213Z","shell.execute_reply":"2026-01-13T10:19:19.924804Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Unaligned transliteration-translation pairs","metadata":{}},{"cell_type":"code","source":"wgt_measures[wgt_measures.isna().any(axis=1)].head(30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.926797Z","iopub.execute_input":"2026-01-13T10:19:19.927535Z","iopub.status.idle":"2026-01-13T10:19:19.940526Z","shell.execute_reply.started":"2026-01-13T10:19:19.927496Z","shell.execute_reply":"2026-01-13T10:19:19.939720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures[wgt_measures.isna().any(axis=1)].index.get_level_values(level=0).nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.941519Z","iopub.execute_input":"2026-01-13T10:19:19.941786Z","iopub.status.idle":"2026-01-13T10:19:19.959943Z","shell.execute_reply.started":"2026-01-13T10:19:19.941757Z","shell.execute_reply":"2026-01-13T10:19:19.958949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"401/train.shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.961085Z","iopub.execute_input":"2026-01-13T10:19:19.961394Z","iopub.status.idle":"2026-01-13T10:19:19.978526Z","shell.execute_reply.started":"2026-01-13T10:19:19.961368Z","shell.execute_reply":"2026-01-13T10:19:19.977729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Missing parts of the translations\nThere are about 250 partially broken translations and repairing them would require finding the specific translations, possibly in the original publications. Since no translation hub database is available for Old Assyrian, this would require to identify the publication first, then retrieve the text and search for a proper translation, then possibly translate it to English (some of them are in German, French, Turkish and others).","metadata":{}},{"cell_type":"code","source":"wgt_measures[wgt_measures['translation_wgt_measures'].isna()].index.get_level_values(level=0).nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:19.979706Z","iopub.execute_input":"2026-01-13T10:19:19.980018Z","iopub.status.idle":"2026-01-13T10:19:20.000434Z","shell.execute_reply.started":"2026-01-13T10:19:19.979982Z","shell.execute_reply":"2026-01-13T10:19:19.999532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures[wgt_measures['translation_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.001757Z","iopub.execute_input":"2026-01-13T10:19:20.002133Z","iopub.status.idle":"2026-01-13T10:19:20.019004Z","shell.execute_reply.started":"2026-01-13T10:19:20.002105Z","shell.execute_reply":"2026-01-13T10:19:20.018103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Missing parts of the transliterations","metadata":{}},{"cell_type":"code","source":"wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.020097Z","iopub.execute_input":"2026-01-13T10:19:20.020760Z","iopub.status.idle":"2026-01-13T10:19:20.036726Z","shell.execute_reply.started":"2026-01-13T10:19:20.020729Z","shell.execute_reply":"2026-01-13T10:19:20.035906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.037912Z","iopub.execute_input":"2026-01-13T10:19:20.038660Z","iopub.status.idle":"2026-01-13T10:19:20.056907Z","shell.execute_reply.started":"2026-01-13T10:19:20.038629Z","shell.execute_reply":"2026-01-13T10:19:20.055854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures[wgt_measures.isna().any(axis=1)].index.get_level_values(level=0).unique().to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.058010Z","iopub.execute_input":"2026-01-13T10:19:20.058404Z","iopub.status.idle":"2026-01-13T10:19:20.075983Z","shell.execute_reply.started":"2026-01-13T10:19:20.058375Z","shell.execute_reply":"2026-01-13T10:19:20.075143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"row_idx=5\ntrain.loc[row_idx, 'transliteration'], train.loc[row_idx, 'translation'], train.loc[row_idx, 'oare_id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.077145Z","iopub.execute_input":"2026-01-13T10:19:20.077428Z","iopub.status.idle":"2026-01-13T10:19:20.088648Z","shell.execute_reply.started":"2026-01-13T10:19:20.077402Z","shell.execute_reply":"2026-01-13T10:19:20.087535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.089752Z","iopub.execute_input":"2026-01-13T10:19:20.090140Z","iopub.status.idle":"2026-01-13T10:19:20.105822Z","shell.execute_reply.started":"2026-01-13T10:19:20.090112Z","shell.execute_reply":"2026-01-13T10:19:20.104913Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drastic measures - possible text landscape with faulty texts removed\nOne approach could be removing the faulty texts altogether and see how the landscape would look like without them.","metadata":{}},{"cell_type":"code","source":"train['transliteration_length']=train['transliteration'].str.split(' ').apply(len)\ntrain['translation_length']=train['translation'].str.split(' ').apply(len)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:20.107004Z","iopub.execute_input":"2026-01-13T10:19:20.107348Z","iopub.status.idle":"2026-01-13T10:19:20.160878Z","shell.execute_reply.started":"2026-01-13T10:19:20.107311Z","shell.execute_reply":"2026-01-13T10:19:20.159909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(15, 5))\n\ntlit_too_short=wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).unique()\ntlan_too_short=wgt_measures[wgt_measures['translation_wgt_measures'].isna()].index.get_level_values(level=0).unique()\nind=tlit_too_short.to_list()\nind.extend(tlan_too_short.to_list())\n\n\naxs[0].scatter(train['transliteration_length'], train['translation_length'], c='gray', alpha=0.3)\naxs[0].scatter(train.loc[wgt_measures[wgt_measures['translation_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy(), 'transliteration_length'],\n            train.loc[wgt_measures[wgt_measures['translation_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy(), 'translation_length'], c='blue', alpha=0.15, label=\"Translation too short\")\naxs[0].scatter(train.loc[wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy(), 'transliteration_length'],\n            train.loc[wgt_measures[wgt_measures['transliteration_wgt_measures'].isna()].index.get_level_values(level=0).unique().to_numpy(), 'translation_length'], c='red', alpha=0.15, label=\"Transliteration too short\")\naxs[0].set_xlabel('Transliteration length (approx.)')\naxs[0].set_ylabel('Translation length (approx.)')\naxs[0].set_title('Transliteration/translation length in words (raw data)')\naxs[0].legend();\n\naxs[1].scatter(train.loc[train.index[~train.index.isin(ind)], 'transliteration_length'], train.loc[train.index[~train.index.isin(ind)], 'translation_length'], c='gray', alpha=0.3)\naxs[1].set_xlabel('Transliteration length (approx.)')\naxs[1].set_ylabel('Translation length (approx.)')\naxs[1].set_title('Transliteration/translation length in words (faulty data removed)');\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T10:19:27.526876Z","iopub.execute_input":"2026-01-13T10:19:27.527214Z","iopub.status.idle":"2026-01-13T10:19:28.022496Z","shell.execute_reply.started":"2026-01-13T10:19:27.527185Z","shell.execute_reply":"2026-01-13T10:19:28.021707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}