{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q sacrebleu\n\nimport re\nimport math\nfrom collections import Counter\nimport numpy as np\nimport pandas as pd\nfrom sacrebleu.metrics import CHRF","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:25:41.544816Z","iopub.execute_input":"2026-01-07T04:25:41.545165Z","iopub.status.idle":"2026-01-07T04:25:41.960432Z","shell.execute_reply.started":"2026-01-07T04:25:41.545122Z","shell.execute_reply":"2026-01-07T04:25:41.959481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sentences_oare_path = \"/kaggle/input/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\"\ntrain_path = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n\nsentences_oare = pd.read_csv(sentences_oare_path)\ntrain = pd.read_csv(train_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:25:49.892369Z","iopub.execute_input":"2026-01-07T04:25:49.892742Z","iopub.status.idle":"2026-01-07T04:25:50.030568Z","shell.execute_reply.started":"2026-01-07T04:25:49.892716Z","shell.execute_reply":"2026-01-07T04:25:50.029612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['transliteration_length'] = train.apply(lambda row: len(row['transliteration'].split(' ')), axis=1)\ntrain['translation_length'] = train.apply(lambda row: len(row['translation'].split(' ')), axis=1)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:25:52.305663Z","iopub.execute_input":"2026-01-07T04:25:52.306007Z","iopub.status.idle":"2026-01-07T04:25:52.380107Z","shell.execute_reply.started":"2026-01-07T04:25:52.305978Z","shell.execute_reply":"2026-01-07T04:25:52.379123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = train.merge(sentences_oare, left_on='oare_id', right_on='text_uuid', suffixes=('_train', '_sentcs'))\nprint(test.shape, len(test['oare_id'].unique()))\ntest.to_csv(\"sentence_match_full.csv\", index=False)\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:34:11.964835Z","iopub.execute_input":"2026-01-07T04:34:11.965177Z","iopub.status.idle":"2026-01-07T04:34:11.989582Z","shell.execute_reply.started":"2026-01-07T04:34:11.965149Z","shell.execute_reply":"2026-01-07T04:34:11.988656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = test.copy()\n\nchrf = CHRF(word_order=2)\n\ndef split_translation_clauses(text):\n    # conservative clause splitting\n    return [t.strip() for t in text.replace(\";\", \".\").split(\".\") if t.strip()]\n\naligned_pairs = []\n\nfor oare_id, g in df.groupby(\"oare_id\"):\n    g = g.sort_values(\"first_word_number\")\n\n    # tokenized transliteration (full tablet)\n    full_translit_tokens = g.iloc[0][\"transliteration\"].split()\n    translation_full = g.iloc[0][\"translation_train\"]\n    translation_clauses = split_translation_clauses(translation_full)\n\n    # sentence boundaries\n    word_starts = g[\"first_word_number\"].tolist()\n    word_starts.append(len(full_translit_tokens) + 1)\n\n    for i, row in g.iterrows():\n        start = row[\"first_word_number\"] - 1\n        end = word_starts[g.index.get_loc(i) + 1] - 1\n\n        translit_sentence = \" \".join(full_translit_tokens[start:end]).strip()\n        anchor = str(row[\"translation_sentcs\"])\n\n        if not translit_sentence or not anchor:\n            continue\n\n        # fuzzy match translation clauses\n        scores = []\n        for j, clause in enumerate(translation_clauses):\n            score = chrf.sentence_score(anchor, [clause]).score\n            scores.append((score, j))\n\n        if not scores:\n            continue\n\n        best_idx = max(scores)[1]\n\n        # try expanding window\n        candidate = translation_clauses[best_idx]\n        if best_idx + 1 < len(translation_clauses):\n            expanded = candidate + \", \" + translation_clauses[best_idx + 1]\n            if chrf.sentence_score(anchor, [expanded]).score > \\\n               chrf.sentence_score(anchor, [candidate]).score:\n                candidate = expanded\n\n        aligned_pairs.append((oare_id, translit_sentence, candidate, anchor))\n\nprint(f\"Aligned sentence pairs: {len(aligned_pairs)}\")\naligned_pairs[:3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T05:04:15.071554Z","iopub.execute_input":"2026-01-07T05:04:15.071920Z","iopub.status.idle":"2026-01-07T05:04:20.046263Z","shell.execute_reply.started":"2026-01-07T05:04:15.071895Z","shell.execute_reply":"2026-01-07T05:04:20.045344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"res = pd.DataFrame(aligned_pairs, columns=[\"oare_id\", \"translit_sent\", \"translation_sent_train\", \"translation_anchor\"])\nres.to_csv(\"sentence_alignment.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T05:06:37.328109Z","iopub.execute_input":"2026-01-07T05:06:37.328620Z","iopub.status.idle":"2026-01-07T05:06:37.351016Z","shell.execute_reply.started":"2026-01-07T05:06:37.328591Z","shell.execute_reply":"2026-01-07T05:06:37.349896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}