{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Initiative – (Data Filtering Notebook)","metadata":{}},{"cell_type":"code","source":"# Imports\n\nimport pandas as pd\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-18T12:32:41.568433Z","iopub.execute_input":"2026-03-18T12:32:41.569278Z","iopub.status.idle":"2026-03-18T12:32:41.897048Z","shell.execute_reply.started":"2026-03-18T12:32:41.569244Z","shell.execute_reply":"2026-03-18T12:32:41.896120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import data\n\nINPUT_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\ntrain_df = pd.read_csv(f\"{INPUT_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{INPUT_DIR}/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-18T12:32:43.239113Z","iopub.execute_input":"2026-03-18T12:32:43.239532Z","iopub.status.idle":"2026-03-18T12:32:43.315946Z","shell.execute_reply.started":"2026-03-18T12:32:43.239503Z","shell.execute_reply":"2026-03-18T12:32:43.315118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find dataset entries with significantly shorter translations\n\n# Calculate difference in length between transliteration and translation\ndiff = (train_df[\"translation\"].str.len() -\n            train_df[\"transliteration\"].str.len())\n\n# Calculate the ratio between the transliteration and translation length\nratio = train_df[\"translation\"].str.len() / train_df[\"transliteration\"].str.len()\n\nD = 500 # Less characters\nR = 5 # Times as small\n\n# Find indices for datapoints with large differences\nmask = (diff < -D) | (ratio < 1/R)\nsmaller = train_df.index[mask]\nprint(f\"{len(smaller)} pairs found where translation is at least {D} characters shorter or {R} times smaller\\n\")\n\n# Inspect outliers\nfor index in smaller:\n    print(f\"index: {index}, difference: {-diff.loc[index]}, ratio: {1/ratio.loc[index]}\")\n    print(f\"{train_df.loc[index]['transliteration']}\\n-->\\n{train_df.loc[index]['translation']}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-18T12:32:45.495812Z","iopub.execute_input":"2026-03-18T12:32:45.496161Z","iopub.status.idle":"2026-03-18T12:32:45.521254Z","shell.execute_reply.started":"2026-03-18T12:32:45.496132Z","shell.execute_reply":"2026-03-18T12:32:45.520458Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find dataset entries where the translations are N larger than the transliteration\n\nD = 1000 # More characters\nR = 3 # Times as large\n\n# Find indices for datapoints with large differences\nmask = (diff > D) & (ratio > R)\nlarger = train_df.index[mask]\nprint(f\"{len(larger)} pairs found where translation is at least {D} characters longer and {R} times larger\\n\")\n\n# Inspect outliers\nfor index in larger:\n    print(f\"index: {index}, difference: {diff.loc[index]}, ratio: {ratio.loc[index]}\")\n    print(f\"{train_df.loc[index]['transliteration']}\\n-->\\n{train_df.loc[index]['translation']}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-18T12:32:49.812704Z","iopub.execute_input":"2026-03-18T12:32:49.813055Z","iopub.status.idle":"2026-03-18T12:32:49.822757Z","shell.execute_reply.started":"2026-03-18T12:32:49.813027Z","shell.execute_reply":"2026-03-18T12:32:49.821776Z"},"jupyter":{"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop data\nindices = smaller.union(larger)\ntrain_df_filtered = train_df.drop(indices)\ntrain_df_filtered.info()\n\n# Save filtered dataset\ntrain_df_filtered.to_csv(\"train_filtered.csv\", index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-18T12:33:59.114660Z","iopub.execute_input":"2026-03-18T12:33:59.115029Z","iopub.status.idle":"2026-03-18T12:33:59.196263Z","shell.execute_reply.started":"2026-03-18T12:33:59.115000Z","shell.execute_reply":"2026-03-18T12:33:59.195305Z"}},"outputs":[],"execution_count":null}]}