{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport math\nfrom collections import Counter\nimport numpy as np\nimport pandas as pd\n\nfrom prettytable import PrettyTable\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:03.568537Z","iopub.execute_input":"2026-01-05T10:50:03.568765Z","iopub.status.idle":"2026-01-05T10:50:03.923177Z","shell.execute_reply.started":"2026-01-05T10:50:03.568741Z","shell.execute_reply":"2026-01-05T10:50:03.922304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sentences_oare_path = \"/kaggle/input/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\"\ntrain_path = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:03.924353Z","iopub.execute_input":"2026-01-05T10:50:03.924743Z","iopub.status.idle":"2026-01-05T10:50:03.928545Z","shell.execute_reply.started":"2026-01-05T10:50:03.924715Z","shell.execute_reply":"2026-01-05T10:50:03.927577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sentences_oare = pd.read_csv(sentences_oare_path)\ntrain = pd.read_csv(train_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:03.929762Z","iopub.execute_input":"2026-01-05T10:50:03.930264Z","iopub.status.idle":"2026-01-05T10:50:04.036809Z","shell.execute_reply.started":"2026-01-05T10:50:03.930238Z","shell.execute_reply":"2026-01-05T10:50:04.035874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['transliteration_length'] = train.apply(lambda row: len(row['transliteration'].split(' ')), axis=1)\ntrain['translation_length'] = train.apply(lambda row: len(row['translation'].split(' ')), axis=1)\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.037973Z","iopub.execute_input":"2026-01-05T10:50:04.038256Z","iopub.status.idle":"2026-01-05T10:50:04.093556Z","shell.execute_reply.started":"2026-01-05T10:50:04.038230Z","shell.execute_reply":"2026-01-05T10:50:04.092635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = train.merge(sentences_oare, left_on='oare_id', right_on='text_uuid', suffixes=('_train', '_sentcs'))\n\n# Replace permutations of whitespace-separated <gap>/<big_gap> permutations before splitting\n# test['collapsed-gap-word-length'] = test.loc[:, 'transliteration'].str.replace(r'((?<!\\S)<(?:gap|big_gap)>(?!\\s?-|\\S)(?:\\s<(?:gap|big_gap)>(?!\\s?-|\\S))+)',\n#                                                                                '<big_gap>', regex=True).apply(lambda x: len(x.split(' ')))\n# test[['oare_id', 'sentence_uuid','first_word_spelling', 'first_word_number', 'original-word-length', 'collapsed-gap-word-length']]\ntest","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.096000Z","iopub.execute_input":"2026-01-05T10:50:04.096272Z","iopub.status.idle":"2026-01-05T10:50:04.120968Z","shell.execute_reply.started":"2026-01-05T10:50:04.096246Z","shell.execute_reply":"2026-01-05T10:50:04.120061Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Problematic transliterations","metadata":{}},{"cell_type":"markdown","source":"## Word mismatch issue","metadata":{}},{"cell_type":"code","source":"matches_validity = test.apply(lambda x: x['transliteration'].split(' ')[x['first_word_number']-1]==x['first_word_spelling'] \\\n                                                                        if x['first_word_number']<=x['transliteration_length'] \\\n                                                                        else None, axis=1)\nCounter(matches_validity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.121950Z","iopub.execute_input":"2026-01-05T10:50:04.122225Z","iopub.status.idle":"2026-01-05T10:50:04.154961Z","shell.execute_reply.started":"2026-01-05T10:50:04.122193Z","shell.execute_reply":"2026-01-05T10:50:04.154096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatched_texts = test.loc[matches_validity==False, :]\nmismatched_texts['oare_id'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.156258Z","iopub.execute_input":"2026-01-05T10:50:04.156740Z","iopub.status.idle":"2026-01-05T10:50:04.171218Z","shell.execute_reply.started":"2026-01-05T10:50:04.156701Z","shell.execute_reply":"2026-01-05T10:50:04.170391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatched_texts['oare_id'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.172431Z","iopub.execute_input":"2026-01-05T10:50:04.172793Z","iopub.status.idle":"2026-01-05T10:50:04.187843Z","shell.execute_reply.started":"2026-01-05T10:50:04.172766Z","shell.execute_reply":"2026-01-05T10:50:04.186981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"TOTAL MISMATCHED TEXTS: {mismatched_texts.shape[0]}\\n\\n\")\nfor ptidx, ptid in zip(mismatched_texts.drop_duplicates(subset='oare_id').index,\n                       mismatched_texts.drop_duplicates(subset='oare_id')['oare_id']):\n    calc_len=test.loc[test['oare_id']==ptid, 'transliteration_length'].values[0]\n    fwn=test.loc[test['oare_id']==ptid, 'first_word_number'].values\n    fws=test.loc[test['oare_id']==ptid, 'first_word_spelling'].values\n    position_words=test.loc[test['oare_id']==ptid, 'transliteration'].str.split(' ').values[0]\n    words_indices = ', '.join([_fws+' ('+str(_fwn-1)+') ' for _fws, _fwn in zip(fws, fwn)])\n\n    print(f\"OARE ID: {ptid} (CALC LEN: {calc_len} WORDS).\\n\")\n    print(f\"\\tTargetted word(s) (expected indices): {words_indices}\")\n    for _fws, _fwn in zip(fws, fwn):\n        print(f\"\\t\\tIdx: {_fwn-1}   Expected: '{_fws}'   Found: '{position_words[_fwn-1] if _fwn<=calc_len else '<idx oob>'}'\\n\")\n    \n    t = PrettyTable(['-']+[str(i) for i in range(10)])\n    print(f\"FULL TEXT:\\n\\t\")\n    for shard in range(0, math.ceil(len(position_words)/10), 1):\n        content=[str(shard)] + position_words[10*(shard):min(10*(shard+1), len(position_words))]\n        if len(content)<11:\n            content.extend([' ' for i in range(11-len(content))])\n        t.add_row(content, divider=True)\n    print(t)\n    print(f\"{'='*20}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.189479Z","iopub.execute_input":"2026-01-05T10:50:04.189826Z","iopub.status.idle":"2026-01-05T10:50:04.237907Z","shell.execute_reply.started":"2026-01-05T10:50:04.189784Z","shell.execute_reply":"2026-01-05T10:50:04.236990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transliterations from `train.csv` are too short","metadata":{}},{"cell_type":"code","source":"trains_too_short = test[matches_validity.isna()]\ntrains_too_short.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.239074Z","iopub.execute_input":"2026-01-05T10:50:04.239451Z","iopub.status.idle":"2026-01-05T10:50:04.270247Z","shell.execute_reply.started":"2026-01-05T10:50:04.239414Z","shell.execute_reply":"2026-01-05T10:50:04.269317Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Problematic texts","metadata":{}},{"cell_type":"code","source":"# first_words=[]\n# problem_index=[]\n# patched_transliterations = test.copy()\n# for fwidx in patched_transliterations.index:\n#     try:\n#         fwnum=patched_transliterations.loc[fwidx, 'first_word_number']\n#         first_words.append( patched_transliterations.loc[fwidx, 'transliteration'].split(' ')[fwnum-1] )\n#         pass\n#     except:\n#         first_words.append(None)\n#         problem_index.append(fwidx)\n# len(problem_index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.271694Z","iopub.execute_input":"2026-01-05T10:50:04.272044Z","iopub.status.idle":"2026-01-05T10:50:04.277514Z","shell.execute_reply.started":"2026-01-05T10:50:04.272009Z","shell.execute_reply":"2026-01-05T10:50:04.276373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trains_too_short.shape[0], trains_too_short['oare_id'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.278720Z","iopub.execute_input":"2026-01-05T10:50:04.278980Z","iopub.status.idle":"2026-01-05T10:50:04.295665Z","shell.execute_reply.started":"2026-01-05T10:50:04.278956Z","shell.execute_reply":"2026-01-05T10:50:04.294898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trains_too_short['oare_id'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.296983Z","iopub.execute_input":"2026-01-05T10:50:04.297361Z","iopub.status.idle":"2026-01-05T10:50:04.313253Z","shell.execute_reply.started":"2026-01-05T10:50:04.297320Z","shell.execute_reply":"2026-01-05T10:50:04.312276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(train.loc[~train['oare_id'].isin(test.loc[matches_validity==True, 'oare_id'].unique()), 'transliteration_length'],\n            train.loc[~train['oare_id'].isin(test.loc[matches_validity==True, 'oare_id'].unique()), 'translation_length'],\n            alpha=0.5,\n            c='grey',\n            label='Valid')\nplt.scatter(train.loc[train['oare_id'].isin(test.loc[matches_validity.isna(), 'oare_id'].unique()), 'transliteration_length'],\n            train.loc[train['oare_id'].isin(test.loc[matches_validity.isna(), 'oare_id'].unique()), 'translation_length'],\n            alpha=0.5,\n            c='red',\n            label='Too short')\nplt.scatter(train.loc[train['oare_id'].isin(test.loc[matches_validity==False, 'oare_id'].unique()), 'transliteration_length'],\n            train.loc[train['oare_id'].isin(test.loc[matches_validity==False, 'oare_id'].unique()), 'translation_length'],\n            alpha=0.5,\n            c='lime',\n            label='Gap normalization issue')\nplt.xlabel('Transliteration length');\nplt.ylabel('Translation length')\nplt.legend();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.314512Z","iopub.execute_input":"2026-01-05T10:50:04.315246Z","iopub.status.idle":"2026-01-05T10:50:04.642077Z","shell.execute_reply.started":"2026-01-05T10:50:04.315208Z","shell.execute_reply":"2026-01-05T10:50:04.640963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"TOTAL PROBLEMATIC TEXTS: {trains_too_short['oare_id'].nunique()}\\n\\n\")\nfor ptidx, ptid in zip(trains_too_short.drop_duplicates(subset='oare_id').index,\n                       trains_too_short.drop_duplicates(subset='oare_id')['oare_id']):\n    calc_len=test.loc[test['oare_id']==ptid, 'transliteration_length'].values[0]\n    fwn=test.loc[test['oare_id']==ptid, 'first_word_number'].values\n    fws=test.loc[test['oare_id']==ptid, 'first_word_spelling'].values\n    position_words=test.loc[test['oare_id']==ptid, 'transliteration'].str.split(' ').values[0]\n    words_indices = ', '.join([_fws+' ('+str(_fwn-1)+') ' for _fws, _fwn in zip(fws, fwn)])\n\n    print(f\"OARE ID: {ptid} (CALC LEN: {calc_len} WORDS).\\n\")\n    # print(f\"\\tTargetted word(s) (expected indices): {words_indices}\")\n    \n    tts = PrettyTable(['IDX', 'Expected', 'Found'])\n    # for shard in range(1, len(position_words)//10):\n    \n    for _fws, _fwn in zip(fws, fwn):\n        tts.add_row([str(_fwn-1), _fws, position_words[_fwn-1] if _fwn<=calc_len else '<Index out of bounds>'], divider=True)\n    print(tts)\n    print(f\"{'='*20}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:50:04.643432Z","iopub.execute_input":"2026-01-05T10:50:04.643830Z","iopub.status.idle":"2026-01-05T10:50:04.749927Z","shell.execute_reply.started":"2026-01-05T10:50:04.643792Z","shell.execute_reply":"2026-01-05T10:50:04.749050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}