{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:21.469113Z","iopub.execute_input":"2025-12-17T12:14:21.469462Z","iopub.status.idle":"2025-12-17T12:14:23.128405Z","shell.execute_reply.started":"2025-12-17T12:14:21.469395Z","shell.execute_reply":"2025-12-17T12:14:23.127289Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Processing and submission","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom collections import defaultdict\n\n# 1. Load data\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\npublished = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\nlexicon = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv')\n\n# 2. Refined Cleaning\ndef official_clean(text):\n    if pd.isna(text): return \"\"\n    text = str(text).lower()\n    text = text.replace('[x]', '<gap>')\n    text = re.sub(r'(\\.\\.\\.|\\[\\.\\.\\. \\.\\.\\.\\])', '<big_gap>', text)\n    text = re.sub(r'[!?/.:˹˺\\[\\]{}]', '', text)\n    return re.sub(r'\\s+', ' ', text).strip()\n\n# 3. Build Master Phrase Table\nprint(\"Building Master Phrase Table...\")\ntrain['clean_src'] = train['transliteration'].apply(official_clean)\n\npotentials = ['translation', 'meaning', 'english', 'target']\npub_trans_col = next((c for c in potentials if c in published.columns), None)\n\nif pub_trans_col:\n    published['clean_src'] = published['transliteration'].apply(official_clean)\n    # Filter to ensure we only take rows that actually have a translation\n    published_clean = published.dropna(subset=['clean_src', pub_trans_col])\n    master_df = pd.concat([\n        train[['clean_src', 'translation']],\n        published_clean[['clean_src', pub_trans_col]].rename(columns={pub_trans_col: 'translation'})\n    ])\nelse:\n    master_df = train[['clean_src', 'translation']]\n\nphrase_table = master_df.dropna().groupby('clean_src')['translation'].agg(lambda x: x.value_counts().index[0]).to_dict()\n\n# 4. Build Lexicon Mapping (Fixed Length Mismatch)\nprint(\"Building Lexicon...\")\n# Clean the lexicon first to avoid the length mismatch error\nlex_clean = lexicon.dropna(subset=['form', 'lexeme']).copy()\nlex_clean['form_lower'] = lex_clean['form'].str.lower()\n# Take the first occurrence if multiple forms exist\nlexicon_map = lex_clean.drop_duplicates('form_lower').set_index('form_lower')['lexeme'].to_dict()\n\n# 5. Winning Logic: Tiered Translation\nglobal_fallback = train['translation'].mode()[0]\n\ndef translate_smart(text, p_dict, l_dict, fallback):\n    cleaned = official_clean(text)\n    if not cleaned: return \"The text is damaged.\"\n    \n    # Tier 1: Exact Phrase (Maximum BLEU)\n    if cleaned in p_dict:\n        return p_dict[cleaned]\n    \n    # Tier 2: High-Confidence Word Match\n    words = cleaned.split()\n    translated = []\n    hits = 0\n    for w in words:\n        if w in l_dict:\n            translated.append(str(l_dict[w]))\n            hits += 1\n        else:\n            # We don't add the untranslated word here to keep the English output clean\n            pass \n            \n    # CRITICAL: Only use word-by-word if we matched at least 60% of the sentence\n    # AND the result isn't suspiciously short.\n    if len(words) > 0:\n        coverage = hits / len(words)\n        if coverage >= 0.6 and len(translated) > 1:\n            return \" \".join(translated)\n    \n    # Tier 3: Safe Fallback\n    return fallback\n\n# 6. Execute\nprint(\"Generating Final Submission...\")\ntest['translation'] = test['transliteration'].apply(lambda x: translate_smart(x, phrase_table, lexicon_map, global_fallback))\n\ntest[['id', 'translation']].to_csv('submission.csv', index=False)\nprint(\"Done! This version is robust and error-free.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:23.130338Z","iopub.execute_input":"2025-12-17T12:14:23.130796Z","iopub.status.idle":"2025-12-17T12:14:24.316482Z","shell.execute_reply.started":"2025-12-17T12:14:23.130765Z","shell.execute_reply":"2025-12-17T12:14:24.315249Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA\n1. Sequence Length & Complexity Analysis","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Calculate lengths\ntest['word_count'] = test['transliteration'].str.split().str.len()\ntrain['word_count'] = train['transliteration'].str.split().str.len()\n\nplt.figure(figsize=(10, 5))\nsns.histplot(train['word_count'], label='Train', kde=True, color='blue', alpha=0.5)\nsns.histplot(test['word_count'], label='Test', kde=True, color='orange', alpha=0.5)\nplt.title(\"Distribution of Sentence Lengths (Word Count)\")\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:24.317836Z","iopub.execute_input":"2025-12-17T12:14:24.318658Z","iopub.status.idle":"2025-12-17T12:14:27.201887Z","shell.execute_reply.started":"2025-12-17T12:14:24.318625Z","shell.execute_reply":"2025-12-17T12:14:27.200527Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2. The \"Gap\" Impact Study","metadata":{}},{"cell_type":"code","source":"def count_gaps(text):\n    return text.count('<gap>') + text.count('<big_gap>')\n\ntest['gap_count'] = test['transliteration'].apply(official_clean).apply(count_gaps)\ntest['damage_ratio'] = test['gap_count'] / test['word_count']\n\n# Plotting\nsns.jointplot(data=test, x='word_count', y='gap_count', kind='hex', cmap='Reds')\nplt.suptitle(\"Sentence Length vs. Text Damage\", y=1.02)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:27.204105Z","iopub.execute_input":"2025-12-17T12:14:27.204589Z","iopub.status.idle":"2025-12-17T12:14:27.531259Z","shell.execute_reply.started":"2025-12-17T12:14:27.204559Z","shell.execute_reply":"2025-12-17T12:14:27.530337Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3. Translation Source Distribution (The \"Tiers\" Audit)","metadata":{}},{"cell_type":"code","source":"def identify_tier(text, p_dict, l_dict):\n    cleaned = official_clean(text)\n    if not cleaned: return \"Empty/Damaged\"\n    if cleaned in p_dict: return \"Tier 1: Phrase Table\"\n    \n    words = cleaned.split()\n    hits = sum(1 for w in words if w in l_dict)\n    coverage = hits / len(words) if len(words) > 0 else 0\n    if coverage >= 0.6 and len(words) > 1: return \"Tier 2: Lexicon\"\n    \n    return \"Tier 3: Fallback\"\n\ntest['translation_tier'] = test['transliteration'].apply(lambda x: identify_tier(x, phrase_table, lexicon_map))\n\n# Visualizing the Tier Distribution\ntier_counts = test['translation_tier'].value_counts()\nplt.figure(figsize=(8, 8))\nplt.pie(tier_counts, labels=tier_counts.index, autopct='%1.1f%%', colors=sns.color_palette('pastel'))\nplt.title(\"Where do your translations come from?\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:27.532682Z","iopub.execute_input":"2025-12-17T12:14:27.533150Z","iopub.status.idle":"2025-12-17T12:14:27.642242Z","shell.execute_reply.started":"2025-12-17T12:14:27.533110Z","shell.execute_reply":"2025-12-17T12:14:27.641282Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4. Lexicon \"Miss\" Analysis (The \"Gold\" Insight)","metadata":{}},{"cell_type":"code","source":"from collections import Counter\n\nall_test_words = [word for text in test['transliteration'].apply(official_clean) for word in text.split()]\nunknown_words = [w for w in all_test_words if w not in lexicon_map and w not in \" \".join(phrase_table.keys())]\n\ntop_unknowns = Counter(unknown_words).most_common(20)\nunknown_df = pd.DataFrame(top_unknowns, columns=['Word', 'Frequency'])\n\nsns.barplot(data=unknown_df, x='Frequency', y='Word')\nplt.title(\"Top 20 Missing Words (Not in Lexicon/Train)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:14:27.643343Z","iopub.execute_input":"2025-12-17T12:14:27.643620Z","iopub.status.idle":"2025-12-17T12:14:28.070030Z","shell.execute_reply.started":"2025-12-17T12:14:27.643594Z","shell.execute_reply":"2025-12-17T12:14:28.068816Z"}},"outputs":[],"execution_count":null}]}