{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport warnings\nwarnings.filterwarnings('ignore')\n\nplt.style.use('seaborn-v0_8-whitegrid')\nsns.set_palette('husl')\n\nDATA_PATH = '/kaggle/input/deep-past-initiative-machine-translation'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:19.523299Z","iopub.execute_input":"2025-12-16T21:13:19.523620Z","iopub.status.idle":"2025-12-16T21:13:20.737155Z","shell.execute_reply.started":"2025-12-16T21:13:19.523599Z","shell.execute_reply":"2025-12-16T21:13:20.735548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Data Overview\n\nLet's start by loading all datasets and understanding their structure.","metadata":{}},{"cell_type":"code","source":"# Load all datasets\ntrain = pd.read_csv(f'{DATA_PATH}/train.csv')\ntest = pd.read_csv(f'{DATA_PATH}/test.csv')\nsample_sub = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\npublished_texts = pd.read_csv(f'{DATA_PATH}/published_texts.csv')\nlexicon = pd.read_csv(f'{DATA_PATH}/OA_Lexicon_eBL.csv')\nbibliography = pd.read_csv(f'{DATA_PATH}/bibliography.csv')\npublications = pd.read_csv(f'{DATA_PATH}/publications.csv')\n\nprint(\"=\"*70)\nprint(\"📊 DATASET OVERVIEW\")\nprint(\"=\"*70)\n\ndatasets = {\n    'train.csv': train,\n    'test.csv': test,\n    'sample_submission.csv': sample_sub,\n    'published_texts.csv': published_texts,\n    'OA_Lexicon_eBL.csv': lexicon,\n    'bibliography.csv': bibliography,\n    'publications.csv': publications\n}\n\nsummary_data = []\nfor name, df in datasets.items():\n    summary_data.append({\n        'Dataset': name,\n        'Rows': f\"{len(df):,}\",\n        'Columns': len(df.columns),\n        'Memory (MB)': f\"{df.memory_usage(deep=True).sum() / 1024**2:.2f}\"\n    })\n\nsummary_df = pd.DataFrame(summary_data)\ndisplay(summary_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:20.739367Z","iopub.execute_input":"2025-12-16T21:13:20.739932Z","iopub.status.idle":"2025-12-16T21:13:29.056382Z","shell.execute_reply.started":"2025-12-16T21:13:20.739901Z","shell.execute_reply":"2025-12-16T21:13:29.055111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visual overview of dataset sizes\nfig, ax = plt.subplots(figsize=(10, 5))\n\nsizes = [len(train), len(test), len(published_texts), len(lexicon), len(bibliography), len(publications)]\nnames = ['Training\\nPairs', 'Test\\nSentences', 'Published\\nTexts', 'Lexicon\\nEntries', 'Bibliography', 'Publication\\nPages']\ncolors = ['#2ecc71', '#e74c3c', '#3498db', '#9b59b6', '#f39c12', '#1abc9c']\n\nbars = ax.bar(names, sizes, color=colors, edgecolor='white', linewidth=2)\n\n# Add value labels\nfor bar, size in zip(bars, sizes):\n    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(sizes)*0.01, \n            f'{size:,}', ha='center', va='bottom', fontsize=11, fontweight='bold')\n\nax.set_ylabel('Number of Records', fontsize=12)\nax.set_title('📊 Dataset Sizes at a Glance', fontsize=14, fontweight='bold')\nax.set_ylim(0, max(sizes) * 1.15)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:29.057470Z","iopub.execute_input":"2025-12-16T21:13:29.057809Z","iopub.status.idle":"2025-12-16T21:13:29.306533Z","shell.execute_reply.started":"2025-12-16T21:13:29.057771Z","shell.execute_reply":"2025-12-16T21:13:29.305307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Training Data Deep Dive\n\nThe training data contains **document-level** parallel pairs of transliterated Akkadian and English translations.","metadata":{}},{"cell_type":"code","source":"print(\"🔍 TRAINING DATA STRUCTURE\")\nprint(\"=\"*70)\nprint(f\"\\nShape: {train.shape}\")\nprint(f\"\\nColumns: {train.columns.tolist()}\")\nprint(f\"\\nData types:\\n{train.dtypes}\")\nprint(f\"\\nNull values: {train.isnull().sum().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:29.307351Z","iopub.execute_input":"2025-12-16T21:13:29.307675Z","iopub.status.idle":"2025-12-16T21:13:29.316426Z","shell.execute_reply.started":"2025-12-16T21:13:29.307650Z","shell.execute_reply":"2025-12-16T21:13:29.315321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show example pairs\nprint(\"\\n📜 EXAMPLE TRANSLATION PAIRS\")\nprint(\"=\"*70)\n\nfor i in [0, 100, 500]:\n    print(f\"\\n{'─'*70}\")\n    print(f\"📄 Document {i} (oare_id: {train.iloc[i]['oare_id'][:8]}...)\")\n    print(f\"{'─'*70}\")\n    print(f\"\\n🔤 TRANSLITERATION:\")\n    print(f\"{train.iloc[i]['transliteration'][:300]}...\")\n    print(f\"\\n🇬🇧 TRANSLATION:\")\n    print(f\"{train.iloc[i]['translation'][:300]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:29.319097Z","iopub.execute_input":"2025-12-16T21:13:29.319458Z","iopub.status.idle":"2025-12-16T21:13:29.340436Z","shell.execute_reply.started":"2025-12-16T21:13:29.319425Z","shell.execute_reply":"2025-12-16T21:13:29.339086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate text statistics\ntrain['translit_chars'] = train['transliteration'].str.len()\ntrain['trans_chars'] = train['translation'].str.len()\ntrain['translit_words'] = train['transliteration'].str.split().str.len()\ntrain['trans_words'] = train['translation'].str.split().str.len()\ntrain['char_ratio'] = train['trans_chars'] / train['translit_chars']\ntrain['word_ratio'] = train['trans_words'] / train['translit_words']\n\n# Statistics table\nstats_df = pd.DataFrame({\n    'Metric': ['Characters', 'Words'],\n    'Transliteration (mean)': [f\"{train['translit_chars'].mean():.0f}\", f\"{train['translit_words'].mean():.0f}\"],\n    'Transliteration (median)': [f\"{train['translit_chars'].median():.0f}\", f\"{train['translit_words'].median():.0f}\"],\n    'Translation (mean)': [f\"{train['trans_chars'].mean():.0f}\", f\"{train['trans_words'].mean():.0f}\"],\n    'Translation (median)': [f\"{train['trans_chars'].median():.0f}\", f\"{train['trans_words'].median():.0f}\"],\n    'Ratio (trans/translit)': [f\"{train['char_ratio'].mean():.2f}x\", f\"{train['word_ratio'].mean():.2f}x\"]\n})\n\nprint(\"\\n📊 TEXT LENGTH STATISTICS\")\ndisplay(stats_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:29.341418Z","iopub.execute_input":"2025-12-16T21:13:29.341747Z","iopub.status.idle":"2025-12-16T21:13:29.412382Z","shell.execute_reply.started":"2025-12-16T21:13:29.341722Z","shell.execute_reply":"2025-12-16T21:13:29.411452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize length distributions\nfig, axes = plt.subplots(2, 2, figsize=(14, 10))\n\n# Character lengths\naxes[0, 0].hist(train['translit_chars'], bins=50, alpha=0.7, color='#3498db', edgecolor='white')\naxes[0, 0].axvline(train['translit_chars'].median(), color='red', linestyle='--', label=f'Median: {train[\"translit_chars\"].median():.0f}')\naxes[0, 0].set_xlabel('Character Length')\naxes[0, 0].set_ylabel('Frequency')\naxes[0, 0].set_title('📝 Transliteration Length (Characters)', fontweight='bold')\naxes[0, 0].legend()\n\naxes[0, 1].hist(train['trans_chars'], bins=50, alpha=0.7, color='#2ecc71', edgecolor='white')\naxes[0, 1].axvline(train['trans_chars'].median(), color='red', linestyle='--', label=f'Median: {train[\"trans_chars\"].median():.0f}')\naxes[0, 1].set_xlabel('Character Length')\naxes[0, 1].set_ylabel('Frequency')\naxes[0, 1].set_title('📝 Translation Length (Characters)', fontweight='bold')\naxes[0, 1].legend()\n\n# Word lengths\naxes[1, 0].hist(train['translit_words'], bins=50, alpha=0.7, color='#9b59b6', edgecolor='white')\naxes[1, 0].axvline(train['translit_words'].median(), color='red', linestyle='--', label=f'Median: {train[\"translit_words\"].median():.0f}')\naxes[1, 0].set_xlabel('Word Count')\naxes[1, 0].set_ylabel('Frequency')\naxes[1, 0].set_title('📊 Transliteration Length (Words)', fontweight='bold')\naxes[1, 0].legend()\n\naxes[1, 1].hist(train['trans_words'], bins=50, alpha=0.7, color='#e74c3c', edgecolor='white')\naxes[1, 1].axvline(train['trans_words'].median(), color='red', linestyle='--', label=f'Median: {train[\"trans_words\"].median():.0f}')\naxes[1, 1].set_xlabel('Word Count')\naxes[1, 1].set_ylabel('Frequency')\naxes[1, 1].set_title('📊 Translation Length (Words)', fontweight='bold')\naxes[1, 1].legend()\n\nplt.tight_layout()\nplt.suptitle('Training Data Length Distributions', fontsize=14, fontweight='bold', y=1.02)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:29.413296Z","iopub.execute_input":"2025-12-16T21:13:29.413537Z","iopub.status.idle":"2025-12-16T21:13:30.741351Z","shell.execute_reply.started":"2025-12-16T21:13:29.413518Z","shell.execute_reply":"2025-12-16T21:13:30.740351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Source vs Target length relationship\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# Scatter plot\naxes[0].scatter(train['translit_words'], train['trans_words'], alpha=0.4, c='#3498db', s=20)\naxes[0].plot([0, 200], [0, 200*train['word_ratio'].mean()], 'r--', \n             label=f'Avg ratio: {train[\"word_ratio\"].mean():.2f}x')\naxes[0].set_xlabel('Transliteration Words')\naxes[0].set_ylabel('Translation Words')\naxes[0].set_title('🔗 Source vs Target Word Count', fontweight='bold')\naxes[0].legend()\n\n# Ratio distribution\naxes[1].hist(train['word_ratio'], bins=50, alpha=0.7, color='#9b59b6', edgecolor='white')\naxes[1].axvline(train['word_ratio'].mean(), color='red', linestyle='--', \n                label=f'Mean: {train[\"word_ratio\"].mean():.2f}x')\naxes[1].set_xlabel('Word Ratio (Translation / Transliteration)')\naxes[1].set_ylabel('Frequency')\naxes[1].set_title('📈 Translation Expansion Ratio', fontweight='bold')\naxes[1].legend()\n\nplt.tight_layout()\nplt.show()\n\nprint(f\"\\n💡 Key Insight: Translations are on average {train['word_ratio'].mean():.2f}x longer than transliterations\")\nprint(f\"   This reflects Akkadian's morphological complexity - one word encodes multiple English words\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:30.742455Z","iopub.execute_input":"2025-12-16T21:13:30.742731Z","iopub.status.idle":"2025-12-16T21:13:31.358162Z","shell.execute_reply.started":"2025-12-16T21:13:30.742705Z","shell.execute_reply":"2025-12-16T21:13:31.356785Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. 🔤 Transliteration Format Analysis\n\nUnderstanding the special characters and patterns in transliterations is **critical** for preprocessing.","metadata":{}},{"cell_type":"code","source":"# Analyze transliteration patterns\nall_translit = ' '.join(train['transliteration'].tolist())\n\nprint(\"🔍 TRANSLITERATION FORMAT ANALYSIS\")\nprint(\"=\"*70)\n\n# 1. Hyphenation\nwords = all_translit.split()\nhyphenated = [w for w in words if '-' in w]\nprint(f\"\\n📌 HYPHENATION:\")\nprint(f\"   Total tokens: {len(words):,}\")\nprint(f\"   Hyphenated tokens: {len(hyphenated):,} ({100*len(hyphenated)/len(words):.1f}%)\")\nprint(f\"   Sample: {hyphenated[:5]}\")\n\n# 2. ALL CAPS (Sumerian logograms)\ncaps_pattern = re.compile(r'\\b[A-ZŠṢṬḪ][A-ZŠṢṬḪ₀-₉\\.]+\\b')\ncaps_words = caps_pattern.findall(all_translit)\ncaps_counts = Counter(caps_words)\nprint(f\"\\n📌 SUMERIAN LOGOGRAMS (ALL CAPS):\")\nprint(f\"   Total occurrences: {len(caps_words):,}\")\nprint(f\"   Unique logograms: {len(caps_counts):,}\")\nprint(f\"   Top 10: {caps_counts.most_common(10)}\")\n\n# 3. Determinatives - check if present\ndeterminatives = re.findall(r'\\{[^}]+\\}', all_translit)\nparens_det = re.findall(r'\\([^)]+\\)', all_translit)\nprint(f\"\\n📌 DETERMINATIVES:\")\nprint(f\"   Curly brackets {{...}}: {len(determinatives)}\")\nprint(f\"   Parentheses (...): {len(parens_det)}\")\nif parens_det:\n    print(f\"   Sample parentheses: {list(set(parens_det))[:10]}\")\n\n# 4. Subscript numbers\nsubscripts = re.findall(r'[a-zšṣṭḫ]+[₀-₉]+', all_translit)\nsubscript_counts = Counter(subscripts)\nprint(f\"\\n📌 SUBSCRIPT NUMBERS:\")\nprint(f\"   Total occurrences: {len(subscripts):,}\")\nprint(f\"   Unique forms: {len(subscript_counts):,}\")\nprint(f\"   Top 10: {subscript_counts.most_common(10)}\")\n\n# 5. Fractions/decimals (representing ancient measures)\nfractions = re.findall(r'\\d+\\.\\d+', all_translit)\nprint(f\"\\n📌 FRACTIONS/DECIMALS:\")\nprint(f\"   Total occurrences: {len(fractions):,}\")\nprint(f\"   Sample values: {list(set(fractions))[:10]}\")\n\n# 6. Special punctuation\nprint(f\"\\n📌 SPECIAL PUNCTUATION:\")\nprint(f\"   Square brackets []: {all_translit.count('[') + all_translit.count(']')}\")\nprint(f\"   Parentheses (): {all_translit.count('(') + all_translit.count(')')}\")\nprint(f\"   Half brackets ˹˺: {all_translit.count('˹') + all_translit.count('˺')}\")\nprint(f\"   Question marks: {all_translit.count('?')}\")\nprint(f\"   Exclamation marks: {all_translit.count('!')}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:31.359279Z","iopub.execute_input":"2025-12-16T21:13:31.359580Z","iopub.status.idle":"2025-12-16T21:13:31.478372Z","shell.execute_reply.started":"2025-12-16T21:13:31.359558Z","shell.execute_reply":"2025-12-16T21:13:31.477197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize Sumerian logogram distribution\ntop_logograms = caps_counts.most_common(20)\n\nfig, ax = plt.subplots(figsize=(12, 6))\nnames = [x[0] for x in top_logograms]\ncounts = [x[1] for x in top_logograms]\n\nbars = ax.barh(names[::-1], counts[::-1], color='#e74c3c', edgecolor='white')\nax.set_xlabel('Frequency', fontsize=12)\nax.set_title('📊 Top 20 Sumerian Logograms in Training Data', fontsize=14, fontweight='bold')\n\n# Add value labels\nfor bar, count in zip(bars, counts[::-1]):\n    ax.text(bar.get_width() + 20, bar.get_y() + bar.get_height()/2, \n            f'{count:,}', ha='left', va='center', fontsize=9)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 Key Logograms Explained:\")\nprint(\"   • KÙ.BABBAR = Silver (most common commodity)\")\nprint(\"   • DUMU = 'son (of)' - patronymic indicator\")\nprint(\"   • KIŠIB = Seal (witness markers)\")\nprint(\"   • IŠTAR = The goddess Ishtar\")\nprint(\"   • AN.NA = Tin (key trade good)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:31.479599Z","iopub.execute_input":"2025-12-16T21:13:31.480489Z","iopub.status.idle":"2025-12-16T21:13:31.871046Z","shell.execute_reply.started":"2025-12-16T21:13:31.480444Z","shell.execute_reply":"2025-12-16T21:13:31.870039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Translation Analysis\n\nLet's analyze the English translations to understand vocabulary and patterns.","metadata":{}},{"cell_type":"code","source":"# Analyze translations\nall_trans = ' '.join(train['translation'].tolist())\n\n# Word frequency\ntrans_words = re.findall(r'\\b[a-z]+\\b', all_trans.lower())\nword_counts = Counter(trans_words)\n\nprint(\"🔍 TRANSLATION VOCABULARY ANALYSIS\")\nprint(\"=\"*70)\nprint(f\"\\n📊 Basic Statistics:\")\nprint(f\"   Total word tokens: {len(trans_words):,}\")\nprint(f\"   Unique words: {len(word_counts):,}\")\nprint(f\"   Vocabulary density: {len(word_counts)/len(trans_words)*100:.2f}%\")\n\nprint(f\"\\n📝 Top 30 Most Common Words:\")\nfor i, (word, count) in enumerate(word_counts.most_common(30), 1):\n    print(f\"   {i:2d}. {word:15s} : {count:,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:31.872574Z","iopub.execute_input":"2025-12-16T21:13:31.873078Z","iopub.status.idle":"2025-12-16T21:13:31.960118Z","shell.execute_reply.started":"2025-12-16T21:13:31.873048Z","shell.execute_reply":"2025-12-16T21:13:31.958849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Proper nouns analysis\nproper_pattern = re.compile(r'\\b[A-ZŠṢṬḪ][a-zšṣṭḫāēīūâêîû]+(?:-[A-Za-zšṣṭḫāēīūâêîû]+)*\\b')\nproper_nouns = proper_pattern.findall(all_trans)\nproper_counts = Counter(proper_nouns)\n\n# Filter out sentence starters\nsentence_starters = {'The', 'If', 'To', 'From', 'He', 'My', 'In', 'As', 'They', 'We', 'You', 'This', 'That', 'When', 'After', 'Before'}\nproper_counts_filtered = {k: v for k, v in proper_counts.items() if k not in sentence_starters}\n\nprint(\"\\n📌 PROPER NOUNS (Names & Places):\")\nprint(f\"   Total occurrences: {len(proper_nouns):,}\")\nprint(f\"   Unique proper nouns: {len(proper_counts_filtered):,}\")\n\n# Top proper nouns\ntop_proper = sorted(proper_counts_filtered.items(), key=lambda x: x[1], reverse=True)[:20]\n\nfig, ax = plt.subplots(figsize=(12, 6))\nnames = [x[0] for x in top_proper]\ncounts = [x[1] for x in top_proper]\n\nbars = ax.barh(names[::-1], counts[::-1], color='#9b59b6', edgecolor='white')\nax.set_xlabel('Frequency', fontsize=12)\nax.set_title('📊 Top 20 Proper Nouns in Translations', fontsize=14, fontweight='bold')\n\nfor bar, count in zip(bars, counts[::-1]):\n    ax.text(bar.get_width() + 5, bar.get_y() + bar.get_height()/2, \n            f'{count:,}', ha='left', va='center', fontsize=9)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 These are primarily merchant names - this was a trading community!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:31.961153Z","iopub.execute_input":"2025-12-16T21:13:31.961422Z","iopub.status.idle":"2025-12-16T21:13:32.403772Z","shell.execute_reply.started":"2025-12-16T21:13:31.961402Z","shell.execute_reply":"2025-12-16T21:13:32.402788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Common translation phrases\nprint(\"\\n📌 COMMON TRANSLATION PATTERNS:\")\nprint(\"=\"*70)\n\ncommon_phrases = [\n    ('son of', 'Patronymic naming'),\n    ('shekels of silver', 'Currency/payment'),\n    ('minas of silver', 'Larger currency amounts'),\n    ('seal of', 'Witness/authentication'),\n    ('witnessed by', 'Legal witnesses'),\n    ('will pay', 'Debt obligations'),\n    ('owes', 'Debt statements'),\n    ('from the week of', 'Time reference'),\n    ('in the eponymy of', 'Year dating'),\n    ('tin', 'Key trade commodity'),\n    ('textiles', 'Key trade commodity'),\n    ('caravan', 'Trade logistics'),\n]\n\nprint(f\"\\n{'Phrase':<25} {'Count':>8}  Purpose\")\nprint(\"-\"*60)\nfor phrase, purpose in common_phrases:\n    count = all_trans.lower().count(phrase.lower())\n    print(f\"{phrase:<25} {count:>8,}  {purpose}\")\n\nprint(\"\\n💡 These patterns reveal the commercial nature of the texts:\")\nprint(\"   • Debt notes and IOUs\")\nprint(\"   • Witnessed contracts\")\nprint(\"   • Trade in silver, tin, and textiles\")\nprint(\"   • Caravan logistics\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.404667Z","iopub.execute_input":"2025-12-16T21:13:32.405055Z","iopub.status.idle":"2025-12-16T21:13:32.522130Z","shell.execute_reply.started":"2025-12-16T21:13:32.404948Z","shell.execute_reply":"2025-12-16T21:13:32.521184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. 🧪 Test Data Analysis\n\n⚠️ **Critical**: The test data has a **different format** than training data!","metadata":{}},{"cell_type":"code","source":"print(\"🧪 TEST DATA STRUCTURE\")\nprint(\"=\"*70)\nprint(f\"\\nShape: {test.shape}\")\nprint(f\"\\nColumns: {test.columns.tolist()}\")\nprint(f\"\\n⚠️  NOTE: This is PLACEHOLDER data. Real test has ~4,000 sentences from ~400 documents.\")\n\ndisplay(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.525490Z","iopub.execute_input":"2025-12-16T21:13:32.525766Z","iopub.status.idle":"2025-12-16T21:13:32.538302Z","shell.execute_reply.started":"2025-12-16T21:13:32.525745Z","shell.execute_reply":"2025-12-16T21:13:32.537146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Highlight the train/test mismatch\nprint(\"\\n⚠️  CRITICAL: TRAIN VS TEST FORMAT MISMATCH\")\nprint(\"=\"*70)\n\ncomparison = pd.DataFrame({\n    'Aspect': ['Alignment Level', 'Avg Length (words)', 'Structure', 'Line Numbers'],\n    'Training Data': ['Document-level', '~58 words', 'Complete tablets', 'Not provided'],\n    'Test Data': ['Sentence-level', '~21 words (est.)', 'Individual sentences', 'line_start, line_end provided']\n})\n\ndisplay(comparison)\n\nprint(\"\\n💡 IMPLICATIONS:\")\nprint(\"   1. You're training on DOCUMENTS but predicting SENTENCES\")\nprint(\"   2. Need strategy for sentence segmentation OR\")\nprint(\"   3. Create synthetic sentence-level training pairs\")\nprint(\"   4. Line numbers in test can help with alignment\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.539288Z","iopub.execute_input":"2025-12-16T21:13:32.540274Z","iopub.status.idle":"2025-12-16T21:13:32.562298Z","shell.execute_reply.started":"2025-12-16T21:13:32.540242Z","shell.execute_reply":"2025-12-16T21:13:32.561096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sample submission format\nprint(\"\\n📝 SUBMISSION FORMAT:\")\nprint(\"=\"*70)\ndisplay(sample_sub)\n\nprint(\"\\n💡 Each test sentence needs ONE English translation.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.563539Z","iopub.execute_input":"2025-12-16T21:13:32.563975Z","iopub.status.idle":"2025-12-16T21:13:32.582351Z","shell.execute_reply.started":"2025-12-16T21:13:32.563923Z","shell.execute_reply":"2025-12-16T21:13:32.581426Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. 📚 Supplementary Data: Published Texts\n\nThis dataset contains **8,000 transliterations** with metadata - but **no translations**. \nUseful for language modeling and understanding the domain.","metadata":{}},{"cell_type":"code","source":"print(\"📚 PUBLISHED TEXTS ANALYSIS\")\nprint(\"=\"*70)\nprint(f\"\\nShape: {published_texts.shape}\")\nprint(f\"\\nColumns: {published_texts.columns.tolist()}\")\n\n# Check overlap with training\ntrain_ids = set(train['oare_id'])\npub_ids = set(published_texts['oare_id'])\noverlap = train_ids & pub_ids\n\nprint(f\"\\n📊 OVERLAP ANALYSIS:\")\nprint(f\"   Training documents: {len(train_ids):,}\")\nprint(f\"   Published texts: {len(pub_ids):,}\")\nprint(f\"   Overlap (in both): {len(overlap):,}\")\nprint(f\"   Additional texts without translations: {len(pub_ids - train_ids):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.583647Z","iopub.execute_input":"2025-12-16T21:13:32.584423Z","iopub.status.idle":"2025-12-16T21:13:32.608162Z","shell.execute_reply.started":"2025-12-16T21:13:32.584389Z","shell.execute_reply":"2025-12-16T21:13:32.606993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Genre distribution\ngenre_counts = published_texts['genre_label'].value_counts()\n\nfig, ax = plt.subplots(figsize=(12, 6))\n\n# Top 15 genres\ntop_genres = genre_counts.head(15)\ncolors = plt.cm.Spectral(np.linspace(0, 1, len(top_genres)))\n\nbars = ax.barh(top_genres.index[::-1], top_genres.values[::-1], color=colors[::-1], edgecolor='white')\nax.set_xlabel('Number of Texts', fontsize=12)\nax.set_title('📊 Genre Distribution of Old Assyrian Texts', fontsize=14, fontweight='bold')\n\nfor bar, count in zip(bars, top_genres.values[::-1]):\n    ax.text(bar.get_width() + 20, bar.get_y() + bar.get_height()/2, \n            f'{count:,}', ha='left', va='center', fontsize=9)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 Key Genres:\")\nprint(\"   • Letters - Personal and business correspondence\")\nprint(\"   • Debt notes - IOUs and financial obligations\")\nprint(\"   • Agreements/Contracts - Legal documents\")\nprint(\"   • Unknown - Many texts haven't been classified\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.609240Z","iopub.execute_input":"2025-12-16T21:13:32.609516Z","iopub.status.idle":"2025-12-16T21:13:32.968920Z","shell.execute_reply.started":"2025-12-16T21:13:32.609488Z","shell.execute_reply":"2025-12-16T21:13:32.967928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compare formatting between train and published_texts\nprint(\"\\n🔍 FORMATTING DIFFERENCES: Train vs Published Texts\")\nprint(\"=\"*70)\n\n# Find a common text\ncommon_id = list(overlap)[0]\ntrain_text = train[train['oare_id'] == common_id]['transliteration'].iloc[0][:200]\npub_text = published_texts[published_texts['oare_id'] == common_id]['transliteration'].iloc[0][:200]\n\nprint(f\"\\nSame document in both datasets:\")\nprint(f\"\\n📄 TRAIN.CSV:\")\nprint(f\"   {train_text}...\")\nprint(f\"\\n📄 PUBLISHED_TEXTS.CSV:\")\nprint(f\"   {pub_text}...\")\n\n# Check for determinatives and gaps\npub_all = ' '.join(published_texts['transliteration'].dropna().tolist())\npub_determinatives = re.findall(r'\\{[^}]+\\}', pub_all)\npub_gaps = re.findall(r'<[^>]+>', pub_all)\n\nprint(f\"\\n📌 Published texts special markers:\")\nprint(f\"   Determinatives {{d}}, {{ki}}, etc.: {len(pub_determinatives):,}\")\nprint(f\"   Gap markers <gap>, <big_gap>: {len(pub_gaps):,}\")\nprint(f\"\\n⚠️  Training data has DIFFERENT formatting:\")\nprint(f\"   - Determinatives as (d) instead of {{d}}\")\nprint(f\"   - No gap markers\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:32.970125Z","iopub.execute_input":"2025-12-16T21:13:32.970513Z","iopub.status.idle":"2025-12-16T21:13:33.014435Z","shell.execute_reply.started":"2025-12-16T21:13:32.970488Z","shell.execute_reply":"2025-12-16T21:13:33.013240Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. 📖 Lexicon Analysis\n\nThe lexicon contains **39,000+ entries** mapping transliterated forms to normalized/lemmatized versions.","metadata":{}},{"cell_type":"code","source":"print(\"📖 LEXICON ANALYSIS\")\nprint(\"=\"*70)\nprint(f\"\\nShape: {lexicon.shape}\")\nprint(f\"\\nColumns: {lexicon.columns.tolist()}\")\n\n# Type distribution\ntype_counts = lexicon['type'].value_counts()\nprint(f\"\\n📊 ENTRY TYPES:\")\nfor entry_type, count in type_counts.items():\n    print(f\"   {entry_type}: {count:,} ({100*count/len(lexicon):.1f}%)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.015585Z","iopub.execute_input":"2025-12-16T21:13:33.016095Z","iopub.status.idle":"2025-12-16T21:13:33.027566Z","shell.execute_reply.started":"2025-12-16T21:13:33.016067Z","shell.execute_reply":"2025-12-16T21:13:33.026263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize lexicon composition\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# Pie chart\ncolors = ['#3498db', '#e74c3c', '#2ecc71']\nexplode = (0.02, 0.02, 0.02)\naxes[0].pie(type_counts.values, labels=type_counts.index, autopct='%1.1f%%',\n            colors=colors, explode=explode, shadow=True, startangle=90)\naxes[0].set_title('📊 Lexicon Entry Types', fontsize=14, fontweight='bold')\n\n# Sample entries\naxes[1].axis('off')\nsample_text = \"📚 Sample Lexicon Entries\\n\\n\"\nsample_text += \"WORDS:\\n\"\nfor _, row in lexicon[lexicon['type'] == 'word'].head(3).iterrows():\n    sample_text += f\"  {row['form']} → {row['lexeme']}\\n\"\nsample_text += \"\\nPERSONAL NAMES (PN):\\n\"\nfor _, row in lexicon[lexicon['type'] == 'PN'].head(3).iterrows():\n    sample_text += f\"  {row['form']} → {row['lexeme']}\\n\"\nsample_text += \"\\nGEOGRAPHIC NAMES (GN):\\n\"\nfor _, row in lexicon[lexicon['type'] == 'GN'].head(3).iterrows():\n    sample_text += f\"  {row['form']} → {row['lexeme']}\\n\"\n\naxes[1].text(0.1, 0.9, sample_text, transform=axes[1].transAxes, fontsize=11,\n             verticalalignment='top', fontfamily='monospace',\n             bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.029006Z","iopub.execute_input":"2025-12-16T21:13:33.029447Z","iopub.status.idle":"2025-12-16T21:13:33.294360Z","shell.execute_reply.started":"2025-12-16T21:13:33.029415Z","shell.execute_reply":"2025-12-16T21:13:33.293124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lexicon coverage of training data\nprint(\"\\n🔍 LEXICON COVERAGE OF TRAINING DATA\")\nprint(\"=\"*70)\n\n# Get unique words from training\ntrain_words = set(all_translit.split())\ntrain_words_lower = set(w.lower() for w in train_words)\n\n# Get lexicon forms\nlex_forms = set(lexicon['form'].dropna().tolist())\nlex_forms_lower = set(f.lower() for f in lex_forms)\n\n# Calculate coverage\nexact_matches = train_words & lex_forms\ncase_matches = train_words_lower & lex_forms_lower\n\nprint(f\"\\n📊 Coverage Statistics:\")\nprint(f\"   Unique tokens in training: {len(train_words):,}\")\nprint(f\"   Lexicon forms: {len(lex_forms):,}\")\nprint(f\"   Exact matches: {len(exact_matches):,} ({100*len(exact_matches)/len(train_words):.1f}%)\")\nprint(f\"   Case-insensitive matches: {len(case_matches):,} ({100*len(case_matches)/len(train_words_lower):.1f}%)\")\n\nprint(f\"\\n💡 The lexicon covers ~70% of training vocabulary\")\nprint(f\"   This is especially useful for proper noun normalization!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.295777Z","iopub.execute_input":"2025-12-16T21:13:33.296165Z","iopub.status.idle":"2025-12-16T21:13:33.363822Z","shell.execute_reply.started":"2025-12-16T21:13:33.296138Z","shell.execute_reply":"2025-12-16T21:13:33.362886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. 📰 Publications Mining Potential\n\nThe `publications.csv` contains OCR'd text from **878 scholarly publications** with embedded translations!","metadata":{}},{"cell_type":"code","source":"print(\"📰 PUBLICATIONS ANALYSIS\")\nprint(\"=\"*70)\nprint(f\"\\nShape: {publications.shape}\")\nprint(f\"\\nColumns: {publications.columns.tolist()}\")\n\n# Basic stats\nprint(f\"\\n📊 Basic Statistics:\")\nprint(f\"   Total pages: {len(publications):,}\")\nprint(f\"   Unique PDFs: {publications['pdf_name'].nunique():,}\")\nprint(f\"   Pages with Akkadian: {publications['has_akkadian'].sum():,} ({100*publications['has_akkadian'].mean():.1f}%)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.364980Z","iopub.execute_input":"2025-12-16T21:13:33.365284Z","iopub.status.idle":"2025-12-16T21:13:33.400929Z","shell.execute_reply.started":"2025-12-16T21:13:33.365261Z","shell.execute_reply":"2025-12-16T21:13:33.399652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sample page content\nprint(\"\\n📄 SAMPLE PAGE CONTENT:\")\nprint(\"=\"*70)\n\n# Find a page with Akkadian\nakkadian_page = publications[publications['has_akkadian'] == True].iloc[0]\nprint(f\"\\nPDF: {akkadian_page['pdf_name'][:60]}...\")\nprint(f\"Page: {akkadian_page['page']}\")\nprint(f\"\\nContent (first 1000 chars):\")\nprint(\"-\"*70)\nprint(akkadian_page['page_text'][:1000])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.402027Z","iopub.execute_input":"2025-12-16T21:13:33.402316Z","iopub.status.idle":"2025-12-16T21:13:33.414444Z","shell.execute_reply.started":"2025-12-16T21:13:33.402289Z","shell.execute_reply":"2025-12-16T21:13:33.412992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Analyze bibliography for languages\nprint(\"\\n📚 BIBLIOGRAPHY ANALYSIS\")\nprint(\"=\"*70)\nprint(f\"\\nTotal publications: {len(bibliography):,}\")\n\n# Year distribution\nbibliography['year_clean'] = pd.to_numeric(bibliography['year'].str[:4], errors='coerce')\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# Year distribution\nyear_counts = bibliography['year_clean'].dropna().astype(int)\naxes[0].hist(year_counts, bins=30, color='#3498db', edgecolor='white', alpha=0.7)\naxes[0].set_xlabel('Publication Year')\naxes[0].set_ylabel('Number of Publications')\naxes[0].set_title('📅 Publication Year Distribution', fontweight='bold')\n\n# Journal distribution\njournal_counts = bibliography['journal'].value_counts().head(10)\naxes[1].barh(journal_counts.index[::-1], journal_counts.values[::-1], color='#9b59b6', edgecolor='white')\naxes[1].set_xlabel('Number of Publications')\naxes[1].set_title('📰 Top 10 Journals', fontweight='bold')\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 The publications span over 100 years of scholarship!\")\nprint(\"   Translations may be in English, German, French, or Turkish.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:33.415743Z","iopub.execute_input":"2025-12-16T21:13:33.416047Z","iopub.status.idle":"2025-12-16T21:13:34.063031Z","shell.execute_reply.started":"2025-12-16T21:13:33.416023Z","shell.execute_reply":"2025-12-16T21:13:34.061892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Estimate potential additional training data\nprint(\"\\n🎯 POTENTIAL DATA MINING OPPORTUNITY\")\nprint(\"=\"*70)\n\npages_with_akkadian = publications['has_akkadian'].sum()\navg_page_len = publications['page_text'].str.len().mean()\n\nprint(f\"\\n📊 Estimation:\")\nprint(f\"   Pages with Akkadian text: {pages_with_akkadian:,}\")\nprint(f\"   Average page length: {avg_page_len:,.0f} characters\")\nprint(f\"   Total Akkadian content: ~{pages_with_akkadian * avg_page_len / 1e6:.1f}M characters\")\n\nprint(f\"\\n💡 IF you can extract translations from these publications:\")\nprint(f\"   - Could potentially 5-10x your training data\")\nprint(f\"   - Translations in multiple languages (need translation)\")\nprint(f\"   - Alignment with published_texts.csv using document IDs\")\nprint(f\"\\n⚠️  This is a MAJOR data engineering challenge but potentially game-changing!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:34.064076Z","iopub.execute_input":"2025-12-16T21:13:34.064533Z","iopub.status.idle":"2025-12-16T21:13:34.166145Z","shell.execute_reply.started":"2025-12-16T21:13:34.064502Z","shell.execute_reply":"2025-12-16T21:13:34.165137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"📌 KEY DATA CHALLENGES\")\nprint(\"=\"*70)\n\nprint(\"\"\"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n1. ULTRA LOW RESOURCE: Only 1,561 training pairs\n2. TRAIN/TEST MISMATCH: Training=documents, Test=sentences\n3. MORPHOLOGICAL COMPLEXITY: 1.56x word expansion ratio\n4. PROPER NOUNS: 13,000+ personal names to handle\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-16T21:13:34.167162Z","iopub.execute_input":"2025-12-16T21:13:34.167398Z","iopub.status.idle":"2025-12-16T21:13:34.173802Z","shell.execute_reply.started":"2025-12-16T21:13:34.167379Z","shell.execute_reply":"2025-12-16T21:13:34.172755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}