{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:41.656630Z","iopub.execute_input":"2026-02-22T18:14:41.656961Z","iopub.status.idle":"2026-02-22T18:14:42.896105Z","shell.execute_reply.started":"2026-02-22T18:14:41.656925Z","shell.execute_reply":"2026-02-22T18:14:42.895089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"firstword = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\")\nfirstword.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:42.897091Z","iopub.execute_input":"2026-02-22T18:14:42.897510Z","iopub.status.idle":"2026-02-22T18:14:43.039615Z","shell.execute_reply.started":"2026-02-22T18:14:42.897473Z","shell.execute_reply":"2026-02-22T18:14:43.038371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"published_text = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\")\npublished_text.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:43.041665Z","iopub.execute_input":"2026-02-22T18:14:43.042014Z","iopub.status.idle":"2026-02-22T18:14:43.371366Z","shell.execute_reply.started":"2026-02-22T18:14:43.041989Z","shell.execute_reply":"2026-02-22T18:14:43.370506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:43.372609Z","iopub.execute_input":"2026-02-22T18:14:43.373152Z","iopub.status.idle":"2026-02-22T18:14:43.425839Z","shell.execute_reply.started":"2026-02-22T18:14:43.373123Z","shell.execute_reply":"2026-02-22T18:14:43.424913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom sklearn.model_selection import train_test_split\n\n# Load your dataset\ndf = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")   # change filename if needed\n\n# ---------------------------\n# Cleaning functions\n# ---------------------------\n\ndef clean_transliteration(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = text.lower()\n\n    # remove gap markers\n    text = re.sub(r\"<gap>|<big_gap>\", \"\", text)\n\n    # remove unwanted symbols\n    text = re.sub(r\"[\\[\\]\\(\\)\\?\\!\\/]\", \"\", text)\n\n    # remove ellipsis\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n\n    # normalize spaces\n    text = re.sub(r\"\\s+\", \" \", text)\n\n    return text.strip()\n\n\ndef clean_translation(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = text.lower()\n\n    # remove gap markers\n    text = re.sub(r\"<gap>\", \"\", text)\n\n    # remove ellipsis\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n\n    # remove brackets\n    text = re.sub(r\"[\\(\\)\\[\\]]\", \"\", text)\n\n    # normalize spaces\n    text = re.sub(r\"\\s+\", \" \", text)\n\n    return text.strip()\n\n\n# ---------------------------\n# Apply cleaning\n# ---------------------------\n\ndf[\"clean_src\"] = df[\"transliteration\"].apply(clean_transliteration)\ndf[\"clean_tgt\"] = df[\"translation\"].apply(clean_translation)\n\n# ---------------------------\n# Remove bad rows\n# ---------------------------\n\ndf = df[(df[\"clean_src\"] != \"\") & (df[\"clean_tgt\"] != \"\")]\n\ndf = df[\n    (df[\"clean_src\"].str.split().str.len() >= 3) &\n    (df[\"clean_tgt\"].str.split().str.len() >= 3)\n]\n\n# ---------------------------\n# Train / Val / Test split\n# ---------------------------\n\ntrain_df, temp_df = train_test_split(df, test_size=0.2, random_state=42)\nval_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42)\n\n# ---------------------------\n# Save final files\n# ---------------------------\n\ntrain_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"train_clean.csv\", index=False)\nval_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"val_clean.csv\", index=False)\ntest_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"test_clean.csv\", index=False)\n\nprint(\"Preprocessing complete!\")\nprint(\"Train:\", len(train_df))\nprint(\"Val:\", len(val_df))\nprint(\"Test:\", len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:43.427074Z","iopub.execute_input":"2026-02-22T18:14:43.427420Z","iopub.status.idle":"2026-02-22T18:14:45.519342Z","shell.execute_reply.started":"2026-02-22T18:14:43.427386Z","shell.execute_reply":"2026-02-22T18:14:45.518281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.sample(5)[[\"clean_src\",\"clean_tgt\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:14:45.520483Z","iopub.execute_input":"2026-02-22T18:14:45.520756Z","iopub.status.idle":"2026-02-22T18:14:45.531813Z","shell.execute_reply.started":"2026-02-22T18:14:45.520732Z","shell.execute_reply":"2026-02-22T18:14:45.530914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Calculate lengths\ndf_raw = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\nraw_lengths = df_raw['transliteration'].str.split().str.len()\nclean_lengths = train_df['clean_src'].str.split().str.len()\n\n# Plotting\nplt.figure(figsize=(12, 6))\nplt.hist(raw_lengths, bins=50, alpha=0.5, label='Raw Data', color='red')\nplt.hist(clean_lengths, bins=50, alpha=0.5, label='Cleaned Data (Min 3 words)', color='blue')\nplt.title('Distribution of Sentence Lengths')\nplt.xlabel('Number of Words')\nplt.ylabel('Frequency')\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T18:15:59.057728Z","iopub.execute_input":"2026-02-22T18:15:59.058687Z","iopub.status.idle":"2026-02-22T18:15:59.499356Z","shell.execute_reply.started":"2026-02-22T18:15:59.058655Z","shell.execute_reply":"2026-02-22T18:15:59.498373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}