{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:28.997453Z","iopub.execute_input":"2025-12-31T10:35:28.997765Z","iopub.status.idle":"2025-12-31T10:35:29.006842Z","shell.execute_reply.started":"2025-12-31T10:35:28.997740Z","shell.execute_reply":"2025-12-31T10:35:29.005731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing\n","metadata":{}},{"cell_type":"code","source":"# importing requried modules for data preocessing\nimport pandas as pd\nimport unicodedata\nimport re\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.009004Z","iopub.execute_input":"2025-12-31T10:35:29.009486Z","iopub.status.idle":"2025-12-31T10:35:29.023402Z","shell.execute_reply.started":"2025-12-31T10:35:29.009420Z","shell.execute_reply":"2025-12-31T10:35:29.022311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_path = '/kaggle/input/deep-past-initiative-machine-translation/train.csv'\ntest_data_path = '/kaggle/input/deep-past-initiative-machine-translation/test.csv'\n\ntrain_df = pd.read_csv(train_data_path)\ntest_df = pd.read_csv(test_data_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.024617Z","iopub.execute_input":"2025-12-31T10:35:29.024894Z","iopub.status.idle":"2025-12-31T10:35:29.088008Z","shell.execute_reply.started":"2025-12-31T10:35:29.024867Z","shell.execute_reply":"2025-12-31T10:35:29.087099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.090030Z","iopub.execute_input":"2025-12-31T10:35:29.090433Z","iopub.status.idle":"2025-12-31T10:35:29.100622Z","shell.execute_reply.started":"2025-12-31T10:35:29.090406Z","shell.execute_reply":"2025-12-31T10:35:29.099462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.101890Z","iopub.execute_input":"2025-12-31T10:35:29.102238Z","iopub.status.idle":"2025-12-31T10:35:29.126836Z","shell.execute_reply.started":"2025-12-31T10:35:29.102201Z","shell.execute_reply":"2025-12-31T10:35:29.125539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Unicode & whitespace normalization\ndef normalize_unicode_and_whitespace(text):\n    if not isinstance(text, str):\n        return \"\"\n\n    text = unicodedata.normalize(\"NFKC\", text)\n    \n    text = text.replace(\"\\n\", \" \").replace(\"\\t\", \" \")\n\n    text = re.sub(r\"\\s+\", \" \", text)\n    \n    text = text.strip()\n    \n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.128262Z","iopub.execute_input":"2025-12-31T10:35:29.128803Z","iopub.status.idle":"2025-12-31T10:35:29.144040Z","shell.execute_reply.started":"2025-12-31T10:35:29.128750Z","shell.execute_reply":"2025-12-31T10:35:29.142971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apply Unicode & whitespace normalization to train data \ntrain_df[\"transliteration_norm\"] = train_df[\"transliteration\"].apply(normalize_unicode_and_whitespace)\ntrain_df[\"translation_norm\"] = train_df[\"translation\"].apply(normalize_unicode_and_whitespace)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.145337Z","iopub.execute_input":"2025-12-31T10:35:29.146115Z","iopub.status.idle":"2025-12-31T10:35:29.317244Z","shell.execute_reply.started":"2025-12-31T10:35:29.146049Z","shell.execute_reply":"2025-12-31T10:35:29.316170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(3):\n    print(\"BEFORE SRC:\", train_df.loc[i, \"transliteration\"])\n    print(\"AFTER  SRC:\", train_df.loc[i, \"transliteration_norm\"])\n    print()\n    print(\"BEFORE TGT:\", train_df.loc[i, \"translation\"])\n    print(\"AFTER  TGT:\", train_df.loc[i, \"translation_norm\"])\n    print(\"=\" * 90)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.318376Z","iopub.execute_input":"2025-12-31T10:35:29.318679Z","iopub.status.idle":"2025-12-31T10:35:29.325213Z","shell.execute_reply.started":"2025-12-31T10:35:29.318652Z","shell.execute_reply":"2025-12-31T10:35:29.324241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apply Unicode & whitespace normalization to test data\ntest_df[\"transliteration_norm\"] = test_df[\"transliteration\"].apply(normalize_unicode_and_whitespace)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.328376Z","iopub.execute_input":"2025-12-31T10:35:29.328717Z","iopub.status.idle":"2025-12-31T10:35:29.345297Z","shell.execute_reply.started":"2025-12-31T10:35:29.328689Z","shell.execute_reply":"2025-12-31T10:35:29.344147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# lower casing function\ndef lowercase_text(text):\n    if not isinstance(text, str):\n        return \"\"\n    return text.lower()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.346698Z","iopub.execute_input":"2025-12-31T10:35:29.347056Z","iopub.status.idle":"2025-12-31T10:35:29.363296Z","shell.execute_reply.started":"2025-12-31T10:35:29.347021Z","shell.execute_reply":"2025-12-31T10:35:29.362291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apply lower casing to train data\ntrain_df[\"transliteration_lc\"] = train_df[\"transliteration_norm\"].apply(lowercase_text)\ntrain_df[\"translation_lc\"] = train_df[\"translation_norm\"].apply(lowercase_text)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.364516Z","iopub.execute_input":"2025-12-31T10:35:29.364836Z","iopub.status.idle":"2025-12-31T10:35:29.401001Z","shell.execute_reply.started":"2025-12-31T10:35:29.364796Z","shell.execute_reply":"2025-12-31T10:35:29.399987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(3):\n    print(\"BEFORE SRC:\", train_df.loc[i, \"transliteration_norm\"])\n    print(\"AFTER  SRC:\", train_df.loc[i, \"transliteration_lc\"])\n    print()\n    print(\"BEFORE TGT:\", train_df.loc[i, \"translation_norm\"])\n    print(\"AFTER  TGT:\", train_df.loc[i, \"translation_lc\"])\n    print(\"=\" * 90)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.402265Z","iopub.execute_input":"2025-12-31T10:35:29.402839Z","iopub.status.idle":"2025-12-31T10:35:29.412034Z","shell.execute_reply.started":"2025-12-31T10:35:29.402810Z","shell.execute_reply":"2025-12-31T10:35:29.411089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apply lower casing to test data\ntest_df[\"transliteration_lc\"] = test_df[\"transliteration_norm\"].apply(lowercase_text)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.413182Z","iopub.execute_input":"2025-12-31T10:35:29.413441Z","iopub.status.idle":"2025-12-31T10:35:29.432396Z","shell.execute_reply.started":"2025-12-31T10:35:29.413418Z","shell.execute_reply":"2025-12-31T10:35:29.431382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split '-' on source only\ndef split_hyphens(text):\n    if not isinstance(text, str):\n        return \"\"\n    # Replace hyphens with space\n    return text.replace(\"-\", \" \")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.433548Z","iopub.execute_input":"2025-12-31T10:35:29.433898Z","iopub.status.idle":"2025-12-31T10:35:29.450596Z","shell.execute_reply.started":"2025-12-31T10:35:29.433863Z","shell.execute_reply":"2025-12-31T10:35:29.449439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"transliteration_hy\"] = train_df[\"transliteration_lc\"].apply(split_hyphens)\ntest_df[\"transliteration_hy\"] = test_df[\"transliteration_lc\"].apply(split_hyphens)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.451689Z","iopub.execute_input":"2025-12-31T10:35:29.451933Z","iopub.status.idle":"2025-12-31T10:35:29.473894Z","shell.execute_reply.started":"2025-12-31T10:35:29.451912Z","shell.execute_reply":"2025-12-31T10:35:29.472906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(3):\n    print(\"BEFORE SRC:\", train_df.loc[i, \"transliteration_lc\"])\n    print(\"AFTER  SRC:\", train_df.loc[i, \"transliteration_hy\"])\n    print(\"=\" * 90)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.475046Z","iopub.execute_input":"2025-12-31T10:35:29.475397Z","iopub.status.idle":"2025-12-31T10:35:29.498477Z","shell.execute_reply.started":"2025-12-31T10:35:29.475361Z","shell.execute_reply":"2025-12-31T10:35:29.497441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count rows containing <gap> or <big_gap>\nnum_gap = train_df[\"transliteration_hy\"].str.contains(\"<gap>\").sum()\nnum_big_gap = train_df[\"transliteration_hy\"].str.contains(\"<big_gap>\").sum()\n\nprint(f\"Number of rows containing <gap>: {num_gap}\")\nprint(f\"Number of rows containing <big_gap>: {num_big_gap}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.499553Z","iopub.execute_input":"2025-12-31T10:35:29.499806Z","iopub.status.idle":"2025-12-31T10:35:29.515332Z","shell.execute_reply.started":"2025-12-31T10:35:29.499783Z","shell.execute_reply":"2025-12-31T10:35:29.514546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Length filtering and Deduplication\n# Tokenize by spaces\ntrain_df[\"src_len\"] = train_df[\"transliteration_hy\"].str.split().apply(len)\n\n# Filter: source length between 3 and 256 tokens\ndf_filtered = train_df[(train_df[\"src_len\"] >= 3) & (train_df[\"src_len\"] <= 256)].copy()\n\nprint(f\"Filtered dataset: {train_df.shape[0]} → {df_filtered.shape[0]} rows\")\n\n\n# Drop exact duplicate source-target pairs\ndf_filtered = df_filtered.drop_duplicates(subset=[\"transliteration_hy\", \"translation_lc\"])\n\nprint(f\"After deduplication: {df_filtered.shape[0]} rows\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.516555Z","iopub.execute_input":"2025-12-31T10:35:29.516828Z","iopub.status.idle":"2025-12-31T10:35:29.572819Z","shell.execute_reply.started":"2025-12-31T10:35:29.516794Z","shell.execute_reply":"2025-12-31T10:35:29.571841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adding <akk> mBART uses language tokens to differentiate source and target languages\n# Akkadian is not included in pretrined launguages, we add a custom tag\ndef add_akkadian_tag(text):\n    if not isinstance(text, str):\n        return \"\"\n    return \"<akk> \" + text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:29.573990Z","iopub.execute_input":"2025-12-31T10:35:29.574370Z","iopub.status.idle":"2025-12-31T10:35:29.579817Z","shell.execute_reply.started":"2025-12-31T10:35:29.574330Z","shell.execute_reply":"2025-12-31T10:35:29.578746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_filtered[\"transliteration_tagged\"] = df_filtered[\"transliteration_hy\"].apply(add_akkadian_tag)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:36.702084Z","iopub.execute_input":"2025-12-31T10:35:36.702613Z","iopub.status.idle":"2025-12-31T10:35:36.709837Z","shell.execute_reply.started":"2025-12-31T10:35:36.702573Z","shell.execute_reply":"2025-12-31T10:35:36.708695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(5):\n    print(df_filtered.loc[i, \"transliteration_hy\"])\n    print(df_filtered.loc[i, \"transliteration_tagged\"])\n    print(\"=\" * 90)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:35:42.695309Z","iopub.execute_input":"2025-12-31T10:35:42.695655Z","iopub.status.idle":"2025-12-31T10:35:42.702352Z","shell.execute_reply.started":"2025-12-31T10:35:42.695626Z","shell.execute_reply":"2025-12-31T10:35:42.701201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[\"transliteration_tagged\"] = test_df[\"transliteration_hy\"].apply(add_akkadian_tag)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:37:42.319646Z","iopub.execute_input":"2025-12-31T10:37:42.319945Z","iopub.status.idle":"2025-12-31T10:37:42.325907Z","shell.execute_reply.started":"2025-12-31T10:37:42.319920Z","shell.execute_reply":"2025-12-31T10:37:42.324890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}