{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\n# Load train and validation sets\ntrain_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\n# Quick look at the first few rows\nprint(train_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:07:07.535457Z","iopub.execute_input":"2026-02-07T19:07:07.535746Z","iopub.status.idle":"2026-02-07T19:07:08.869962Z","shell.execute_reply.started":"2026-02-07T19:07:07.535713Z","shell.execute_reply":"2026-02-07T19:07:08.869293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Number of rows and columns\nprint(train_df.shape)\n\n# Check missing values\nprint(train_df.isnull().sum())\n\n# Check unique IDs\nprint(train_df['oare_id'].nunique())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:07:08.871792Z","iopub.execute_input":"2026-02-07T19:07:08.872334Z","iopub.status.idle":"2026-02-07T19:07:08.880719Z","shell.execute_reply.started":"2026-02-07T19:07:08.872308Z","shell.execute_reply":"2026-02-07T19:07:08.879781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add length columns\ntrain_df['trans_len'] = train_df['transliteration'].apply(lambda x: len(str(x).split()))\ntrain_df['eng_len'] = train_df['translation'].apply(lambda x: len(str(x).split()))\n\n# Basic statistics\nprint(train_df[['trans_len', 'eng_len']].describe())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:07:08.881808Z","iopub.execute_input":"2026-02-07T19:07:08.882139Z","iopub.status.idle":"2026-02-07T19:07:08.933740Z","shell.execute_reply.started":"2026-02-07T19:07:08.882105Z","shell.execute_reply":"2026-02-07T19:07:08.932880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter\n\n# Faster token counting\ntrans_tokens = Counter(token for row in train_df['transliteration'] for token in str(row).split())\neng_tokens = Counter(token for row in train_df['translation'] for token in str(row).split())\n\nprint(f\"Unique transliteration tokens: {len(trans_tokens)}\")\nprint(f\"Unique English tokens: {len(eng_tokens)}\")\nprint(\"Most common transliteration tokens:\", trans_tokens.most_common(20))\nprint(\"Most common English tokens:\", eng_tokens.most_common(20))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:07:09.050900Z","iopub.execute_input":"2026-02-07T19:07:09.051269Z","iopub.status.idle":"2026-02-07T19:07:09.130849Z","shell.execute_reply.started":"2026-02-07T19:07:09.051206Z","shell.execute_reply":"2026-02-07T19:07:09.130055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df['transliteration'].str.contains('<gap>').sum())\nprint(train_df['transliteration'].str.contains('<big_gap>').sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:07:09.131869Z","iopub.execute_input":"2026-02-07T19:07:09.132267Z","iopub.status.idle":"2026-02-07T19:07:09.140689Z","shell.execute_reply.started":"2026-02-07T19:07:09.132212Z","shell.execute_reply":"2026-02-07T19:07:09.139933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Distribution of lengths\nplt.figure(figsize=(12,5))\nsns.histplot(train_df['trans_len'], color='blue', label='Transliteration', bins=50, kde=True)\nsns.histplot(train_df['eng_len'], color='red', label='English', bins=50, kde=True)\nplt.legend()\nplt.xlabel(\"Sentence length (words)\")\nplt.ylabel(\"Frequency\")\nplt.title(\"Sentence Length Distribution\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:08:31.835114Z","iopub.execute_input":"2026-02-07T19:08:31.835615Z","iopub.status.idle":"2026-02-07T19:08:33.158907Z","shell.execute_reply.started":"2026-02-07T19:08:31.835588Z","shell.execute_reply":"2026-02-07T19:08:33.158299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsns.scatterplot(x='trans_len', y='eng_len', data=train_df, alpha=0.3)\nplt.xlabel(\"Transliteration length\")\nplt.ylabel(\"English translation length\")\nplt.title(\"Source vs Target Length\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:09:31.005940Z","iopub.execute_input":"2026-02-07T19:09:31.006829Z","iopub.status.idle":"2026-02-07T19:09:31.196303Z","shell.execute_reply.started":"2026-02-07T19:09:31.006798Z","shell.execute_reply":"2026-02-07T19:09:31.195699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Unique transliteration tokens:\", len(trans_tokens))\nprint(\"Unique English tokens:\", len(eng_tokens))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:10:26.913022Z","iopub.execute_input":"2026-02-07T19:10:26.913355Z","iopub.status.idle":"2026-02-07T19:10:26.917632Z","shell.execute_reply.started":"2026-02-07T19:10:26.913326Z","shell.execute_reply":"2026-02-07T19:10:26.916973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"short_translations = train_df[train_df['eng_len'] < 3]\nprint(\"Lines with very short English translations:\", len(short_translations))\nprint(short_translations.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:12:02.269676Z","iopub.execute_input":"2026-02-07T19:12:02.270333Z","iopub.status.idle":"2026-02-07T19:12:02.277098Z","shell.execute_reply.started":"2026-02-07T19:12:02.270286Z","shell.execute_reply":"2026-02-07T19:12:02.276416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Correlation between transliteration and English lengths:\",\n      train_df['trans_len'].corr(train_df['eng_len']))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-07T19:12:12.840681Z","iopub.execute_input":"2026-02-07T19:12:12.841362Z","iopub.status.idle":"2026-02-07T19:12:12.846334Z","shell.execute_reply.started":"2026-02-07T19:12:12.841335Z","shell.execute_reply":"2026-02-07T19:12:12.845678Z"}},"outputs":[],"execution_count":null}]}