{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 1. Install (Kaggle-safe)\n!pip install -q --no-cache-dir algoliasearch==4.35.4\n\n# 2. Correct Imports for v4\n# Using SearchClientSync for standard synchronous code\nfrom algoliasearch.search.client import SearchClientSync\n\ntry:\n    # Initialize the client (Replace with your actual keys)\n    client = SearchClientSync(\"YOUR_APP_ID\", \"YOUR_API_KEY\")\n    \n    print(\"=== FULL SUCCESS ===\")\n    print(\"Algolia v4 Sync Client initialized successfully!\")\n    \nexcept Exception as e:\n    print(f\"Initialization failed: {e}\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-24T03:58:09.478856Z","iopub.execute_input":"2026-01-24T03:58:09.479226Z","iopub.status.idle":"2026-01-24T03:58:13.336454Z","shell.execute_reply.started":"2026-01-24T03:58:09.479187Z","shell.execute_reply":"2026-01-24T03:58:13.335251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom kaggle_secrets import UserSecretsClient\nfrom algoliasearch.search.client import SearchClientSync\n\n# 1. Correct the path to a specific FILE\nBASE_PATH = '/kaggle/input/deep-past-initiative-machine-translation/'\n# Verify files in the directory if you are unsure of the name\nprint(os.listdir(BASE_PATH)) \n\n# Load the desired CSV (e.g., train.csv or OA_Lexicon_eBL.csv)\ndict_df = pd.read_csv(os.path.join(BASE_PATH, 'train.csv')) \n\n# 2. Algolia Setup\nuser_secrets = UserSecretsClient()\nAPP_ID = user_secrets.get_secret(\"ALGOLIA_APP_ID\")\nAPI_KEY = user_secrets.get_secret(\"ALGOLIA_ADMIN_KEY\")\nclient = SearchClientSync(APP_ID, API_KEY)\n\n# 3. Convert and Upload\nrecords = []\nfor idx, row in dict_df.iterrows():\n    # Note: Ensure these functions are defined in your notebook\n    meaning = extract_core_meaning(row['definition']) if 'definition' in row else \"\"\n    field = get_semantic_field(meaning) if (meaning and 'get_semantic_field' in globals()) else \"general\"\n    \n    records.append({\n        \"objectID\": str(idx),\n        \"word\": row.get('word', row.get('transliteration', '')), # Handles different column names\n        \"meaning\": meaning,\n        \"field\": field,\n        \"raw_definition\": row.get('definition', '')\n    })\n\nclient.save_objects(index_name=\"YOUR_INDEX_NAME\", objects=records)\nprint(f\"Successfully uploaded {len(records)} records.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T04:07:06.623346Z","iopub.execute_input":"2026-01-24T04:07:06.623728Z","iopub.status.idle":"2026-01-24T04:07:09.762603Z","shell.execute_reply.started":"2026-01-24T04:07:06.623699Z","shell.execute_reply":"2026-01-24T04:07:09.761673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install the latest Algolia v4 client (mandatory for SearchClientSync)\n!pip install -q --no-cache-dir algoliasearch==4.35.4\n\nimport pandas as pd\nimport re\nfrom pathlib import Path\nfrom algoliasearch.search.client import SearchClientSync\nfrom kaggle_secrets import UserSecretsClient\n\n# Load Kaggle Secrets\nuser_secrets = UserSecretsClient()\nAPP_ID = user_secrets.get_secret(\"ALGOLIA_APP_ID\")\nAPI_KEY = user_secrets.get_secret(\"ALGOLIA_ADMIN_KEY\")\n\n# Competition Data Paths\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nclient = SearchClientSync(APP_ID, API_KEY)\nINDEX_NAME = \"akkadian_lexicon_v2\"\n\n# COMPREHENSIVE SUMEROGRAM LOOKUP TABLE\nSUMEROGRAMS = {\n    'PN': 'personal name',\n    'GN': 'place name',   \n    'KIŠIB': 'seal of', 'DUMU': 'son of', 'GÍN': 'shekel', 'MA.NA': 'mina',\n    'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper', 'AN.NA': 'tin',\n    'TÚG': 'textile', 'IGI': 'before', 'ITI': 'month', 'ITI.KAM': 'month',\n    'MU': 'year', 'UD': 'day', 'É': 'house', 'É.GAL': 'palace', 'DINGIR': 'god',\n    'SIG₅': 'good quality', 'KÙ.AN': 'meteoric iron', 'DAM.GÀR': 'merchant',\n    'aa': 'not', 'ia-tù': 'mine', 'aé-bi„-lá': 'bring!', 'ta-áa-me-a-ni': 'you hear',\n    'É.GAL-lim': 'palace', 'a-ma-kam': 'there', 'qí-bi„-ma': 'speak!',\n    'wa-bar-ra-tim': 'colony', 'kà-ni-ia-ma': 'Kanesh', 'né-mì-lim': 'profit',\n    'a-nim': 'this', 'da-aùr': 'Assur', 'a-lim(ki)': 'City (Ashur)',\n    'a-na': 'to', 'i-na': 'in', 'ša': 'of/which', 'ù': 'and', 'u': 'and',\n    'um-ma': 'thus', 'šu-ma': 'if', 'lá': 'not', 'u-la': 'not', 'ú-lá': 'not',\n    'lu': 'may/or', 'a-dí': 'until', 'iš-tù': 'from', 'ki-ma': 'like/when',\n    'ma-na': 'mina', 'ma-na-im': 'mina', 'GÍN.TA': 'shekels each', 'GÚ': 'talent',\n    'me-at': 'hundred', 'ANŠE': 'donkey', 'ku-ta-nu': 'textiles',\n    'qí-bi-ma': 'speak!', 'qí-bi₄-ma': 'speak!', 'li-mu-um': 'eponym',\n    '1': '1', '2': '2', '3': '3', '4': '4', '5': '5', '6': '6', '7': '7',\n    '8': '8', '9': '9', '10': '10', '0.33333': '1/3', '0.5': '1/2', '1.5': '1.5',\n    '…': '[...]', 'x': '[x]', 'ú': 'and',\n    'mup-pì-im': 'tablet',\n    'mup-pu-um': 'tablet',\n}\n\n# EXPANDED CONTEXT ANCHORS\nCONTEXT_ANCHORS = {\n    'commerce': ['GÍN', 'KÙ.BABBAR', 'ŠE', 'ŠÁM', 'nadānum', 'mahārum', \n                 'šīmum', 'kasap', 'silver', 'barley', 'grain', 'MA.NA', \n                 'mina', 'KÙ.GI', 'gold', 'URUDU', 'copper', 'AN.NA', 'tin',\n                 'DAM.GÀR', 'merchant', 'ku-ta-nu', 'textiles'],\n    'legal': ['IGI', 'KIŠIB', 'kunukkum', 'dīnum', 'awātum', 'šībum',\n              'witness', 'seal', 'judge', 'oath', 'tablet'],\n    'kinship': ['DUMU', 'DAM', 'AḪ', 'ahum', 'mār', 'aššatum', \n                'son', 'brother', 'wife', 'father']\n}\n\n\ndef normalize_akkadian(text):\n    \"\"\"Remove diacritics and normalize Akkadian orthography for matching\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text).lower()\n    \n    # Diacritic normalization\n    replacements = {\n        'š': 's', 'ṣ': 's', 'ś': 's',\n        'ṭ': 't', 'ṯ': 't',\n        'ḫ': 'h', 'ḥ': 'h',\n        'ā': 'a', 'â': 'a', 'á': 'a', 'à': 'a',\n        'ē': 'e', 'ê': 'e', 'é': 'e', 'è': 'e',\n        'ī': 'i', 'î': 'i', 'í': 'i', 'ì': 'i',\n        'ū': 'u', 'û': 'u', 'ú': 'u', 'ù': 'u',\n        '₂': '2', '₃': '3', '₄': '4', '₅': '5',\n        '„': '', '\"': '', '\"': '', '\"': '',\n    }\n    \n    for old, new in replacements.items():\n        text = text.replace(old, new)\n    \n    return text.strip()\n\n\ndef ingest_dictionary():\n    print(\"\\n\" + \"=\"*80)\n    print(\"STARTING DICTIONARY INGESTION\")\n    print(\"=\"*80)\n    \n    dict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\n    lexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\n    \n    print(f\"\\nLoaded {len(dict_df)} entries from eBL_Dictionary.csv\")\n    print(f\"Loaded {len(lexicon_df)} entries from OA_Lexicon_eBL.csv\")\n    \n    records = []\n    \n    # Process eBL dictionary entries\n    print(\"\\n--- Processing eBL Dictionary ---\")\n    field_counts = {'commerce': 0, 'legal': 0, 'kinship': 0, 'general': 0}\n    grammar_counts = {'deity': 0, 'place': 0, 'noun': 0}\n    \n    for idx, row in dict_df.iterrows():\n        word = str(row.get('word', '')).strip().lower()\n        defn = str(row.get('definition', ''))\n        \n        # Skip empty entries\n        if not word or word == 'nan':\n            continue\n        \n        # Detect Semantic Field\n        field = \"general\"\n        if any(term in defn.lower() for term in ['silver', 'shekel', 'mina', 'pay', 'buy', 'sell']):\n            field = \"commerce\"\n        elif any(term in defn.lower() for term in ['seal', 'witness', 'judge', 'oath', 'tablet']):\n            field = \"legal\"\n        elif any(term in defn.lower() for term in ['son', 'daughter', 'father', 'mother', 'brother', 'wife']):\n            field = \"kinship\"\n        \n        field_counts[field] += 1\n        \n        # Detect Grammar Type\n        grammar_type = \"noun\"\n        if \"{d}\" in word or \"god\" in defn.lower(): \n            grammar_type = \"deity\"\n        if \"{ki}\" in word or \"city\" in defn.lower(): \n            grammar_type = \"place\"\n        \n        grammar_counts[grammar_type] += 1\n\n        # Store both original and normalized forms\n        records.append({\n            \"objectID\": f\"dict_{idx}\",\n            \"word\": word,\n            \"word_normalized\": normalize_akkadian(word),\n            \"meaning\": defn,\n            \"field\": field,\n            \"grammar_type\": grammar_type,\n            \"source\": \"eBL\",\n            'priority': 1\n        })\n        \n        if idx < 3:\n            print(f\"  [{idx}] {word[:30]:30s} → {defn[:50]:50s} | field={field}, type={grammar_type}\")\n    \n    print(f\"\\n  Field distribution: {field_counts}\")\n    print(f\"  Grammar distribution: {grammar_counts}\")\n    \n    # Process OA_Lexicon entries (FIXED COLUMNS)\n    print(\"\\n--- Processing OA_Lexicon ---\")\n    oa_field_counts = {'commerce': 0, 'legal': 0, 'kinship': 0, 'general': 0}\n    \n    for idx, row in lexicon_df.iterrows():\n        # Use 'form' as the word and 'lexeme' as the meaning\n        word = str(row.get('form', '')).strip().lower()\n        lexeme = str(row.get('lexeme', '')).strip()\n        \n        # Skip empty entries\n        if not word or word == 'nan' or not lexeme or lexeme == 'nan':\n            continue\n        \n        # Basic field detection (we don't have definitions, so this will mostly be general)\n        field = \"general\"\n        if any(term in lexeme.lower() for term in ['silver', 'shekel', 'mina', 'kasap', 'sāmum']):\n            field = \"commerce\"\n        elif any(term in lexeme.lower() for term in ['seal', 'witness', 'judge', 'dīnum']):\n            field = \"legal\"\n        elif any(term in lexeme.lower() for term in ['son', 'mār', 'ahum', 'aššatum']):\n            field = \"kinship\"\n        \n        oa_field_counts[field] += 1\n        \n        records.append({\n            \"objectID\": f\"oa_lex_{idx}\",\n            \"word\": word,\n            \"word_normalized\": normalize_akkadian(word),\n            \"meaning\": lexeme,  # Use lexeme as meaning\n            \"field\": field,\n            \"grammar_type\": \"noun\",\n            \"source\": \"OA_Lexicon\",\n            \"priority\": 2\n        })\n        \n        if idx < 3:\n            print(f\"  [{idx}] {word[:30]:30s} → {lexeme[:50]:50s} | field={field}\")\n    \n    print(f\"\\n  Field distribution: {oa_field_counts}\")\n    \n    # Batch upload to Algolia\n    print(\"\\n--- Uploading to Algolia ---\")\n    client.save_objects(index_name=INDEX_NAME, objects=records)\n    print(f\"Successfully ingested {len(records)} total records into Algolia\")\n    print(f\"   - {sum(1 for r in records if r['source'] == 'eBL')} from eBL Dictionary\")\n    print(f\"   - {sum(1 for r in records if r['source'] == 'OA_Lexicon')} from OA_Lexicon\")\n    print(\"=\"*80 + \"\\n\")\n\n# Run this once per session\ningest_dictionary()\n\n\ndef translate_token_with_algolia(token, current_context, doc_type, verbose=True, confidence_threshold=5):\n    \"\"\"\n    Finds the best-fit translation by boosting matches that share the \n    current sentence's semantic field (e.g., 'commerce').\n    \"\"\"\n    \n    if verbose:\n        print(f\"\\n  Translating: '{token}'\")\n    \n    # Check Sumerograms first\n    if token in SUMEROGRAMS:\n        result = SUMEROGRAMS[token]\n        if verbose:\n            print(f\"    SUMEROGRAM MATCH: {token} → {result}\")\n        return result\n    \n    # Strip determinatives\n    clean_token = re.sub(r'\\{[^}]+\\}', '', token)\n    if clean_token != token and verbose:\n        print(f\"    → Stripped determinatives: '{token}' → '{clean_token}'\")\n    \n    # Normalize for searching\n    # Normalize for searching\n    normalized_token = normalize_akkadian(clean_token)\n    \n    # Strip common Akkadian suffixes (only complete final syllables)\n    akkadian_suffixes = ['um', 'im', 'am', 'ma', 'ni', 'su', 'si', 'ka', 'ki', 'ti', 'ta', 'tu']\n    root_match = normalized_token\n    search_both = False\n    \n    # Only strip if token has hyphens (syllable boundaries)\n    if '-' in normalized_token:\n        parts = normalized_token.split('-')\n        if len(parts) > 1 and parts[-1] in akkadian_suffixes:\n            root_match = '-'.join(parts[:-1])  # Remove only the last syllable\n            search_both = True\n            if verbose:\n                print(f\"    -> Stripped final syllable '-{parts[-1]}': '{normalized_token}' -> '{root_match}'\")\n    \n    # Smarter root extraction\n    prefix = None\n    if re.match(r'^(a|ta|i|ni)-[a-z]{3,}', normalized_token):\n        prefix = normalized_token.split('-')[0]\n        root_match = '-'.join(normalized_token.split('-')[1:])\n        if verbose:\n            print(f\"    → Detected verb prefix: '{prefix}-' | root: '{root_match}'\")\n    elif verbose:\n        print(f\"    → No verb prefix detected, searching as-is\")\n    \n    search_params = {\n        \"query\": root_match,\n        \"hitsPerPage\": 3,\n        \"typoTolerance\": \"strict\",  # Changed from \"min\" to allow more fuzzy matching\n        \"exactOnSingleWordQuery\": \"word\",\n        \"optionalFilters\": [\n            f\"field:{current_context}<score=10>\",\n            f\"grammar_type:{doc_type}<score=5>\",\n            f\"source:OA_Lexicon<score=15>\"\n        ]\n    }\n    \n    if prefix:\n        search_params[\"optionalFilters\"].append(f\"prefix:{prefix}<score=3>\")\n    \n    if verbose:\n        print(f\"    → Search query: '{root_match}' | context: {current_context}\")\n    \n    try:\n        response = client.search_single_index(index_name=INDEX_NAME, search_params=search_params)\n        \n        if response.hits and len(response.hits) > 0:\n            if verbose:\n                print(f\"    → Found {len(response.hits)} matches:\")\n                for i, hit in enumerate(response.hits[:3]):\n                    hit_dict = hit.to_dict() if hasattr(hit, 'to_dict') else dict(hit)\n                    word = hit_dict.get('word', '')\n                    meaning = hit_dict.get('meaning', '')\n                    source = hit_dict.get('source', '')\n                    print(f\"       [{i+1}] {word[:20]:20s} → {meaning[:40]:40s} | source={source}\")\n\n                # Search through ALL results for exact match (pick shortest if multiple)\n            exact_matches = []\n            for hit in response.hits:\n                hit_dict = hit.to_dict() if hasattr(hit, 'to_dict') else dict(hit)\n                matched_normalized = hit_dict.get('word_normalized', '')\n                matched_word = hit_dict.get('word', '')\n                \n                # Check for exact match\n                # Check for exact match against BOTH stripped and original\n                if (matched_normalized == root_match or \n                    matched_word == clean_token or \n                    (search_both and (matched_normalized == normalized_token or matched_word == normalized_token))):\n                    exact_matches.append((len(matched_word), hit_dict))\n            # If exact matches found, use the shortest one\n            if exact_matches:\n                exact_matches.sort(key=lambda x: x[0])  # Sort by length\n                best_match = exact_matches[0][1]\n                if verbose:\n                    print(f\"    -> Found EXACT MATCH: '{best_match.get('word', '')}'\")\n            else:\n                # Use Algolia's top result\n                best_match = response.hits[0].to_dict() if hasattr(response.hits[0], 'to_dict') else dict(response.hits[0])\n                if verbose:\n                    print(f\"    -> No exact match, using top fuzzy result\")\n            \n            # Get the matched info for comparison\n            matched_word = best_match.get('word', '')\n            matched_normalized = best_match.get('word_normalized', '')\n            meaning = best_match.get('meaning', clean_token)\n\n            # Check if it's an exact match\n           \n            is_exact_match = (\n                matched_normalized == root_match or \n                matched_word == clean_token\n            )            \n            # Better grammatical reconstruction\n            original_meaning = meaning\n            if prefix == 'i' and not meaning.startswith('he'):\n                meaning = f\"he/it {meaning}\"\n            elif prefix == 'a' and not meaning.startswith('I'):\n                meaning = f\"I {meaning}\"\n            elif prefix == 'ta':\n                meaning = f\"you {meaning}\"\n            elif prefix == 'ni':\n                meaning = f\"we {meaning}\"\n            \n            if verbose and original_meaning != meaning:\n                print(f\"    -> Applied prefix grammar: '{original_meaning}' -> '{meaning}'\")\n            \n            # Add asterisk only for fuzzy matches\n            if not is_exact_match:\n                meaning = f\"{meaning}*\"\n                if verbose:\n                    print(f\"    -> FUZZY MATCH (searched: '{root_match}', got: '{matched_normalized}'), adding asterisk\")\n            else:\n                if verbose:\n                    print(f\"    -> EXACT MATCH, no asterisk\")\n            \n            if verbose:\n                print(f\"    FINAL: '{token}' -> '{meaning}'\")\n            \n            return meaning\n        else:\n            if verbose:\n                print(f\"    NO MATCHES FOUND - returning original token\")\n    except Exception as e:\n        if verbose:\n            print(f\"    SEARCH ERROR: {str(e)}\")\n        return clean_token\n    \n    return clean_token\n\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"STARTING TRANSLATION\")\nprint(\"=\"*80)\n\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nprint(f\"\\nLoaded {len(test_df)} test sentences\\n\")\n\nresults = []\n\nfor row_idx, row in test_df.iterrows():\n    print(f\"\\n{'─'*80}\")\n    print(f\"SENTENCE {row_idx + 1}/{len(test_df)} | ID: {row['id']}\")\n    print(f\"{'─'*80}\")\n    \n    tokens = row['transliteration'].split()\n    print(f\"Original: {row['transliteration']}\")\n    print(f\"Tokens ({len(tokens)}): {tokens}\\n\")\n    \n    # Better context detection\n    context_scores = {ctx: 0 for ctx in CONTEXT_ANCHORS.keys()}\n    matched_anchors = {ctx: [] for ctx in CONTEXT_ANCHORS.keys()}\n    \n    for token in tokens:\n        for ctx, anchors in CONTEXT_ANCHORS.items():\n            if token in anchors:\n                context_scores[ctx] += 1\n                matched_anchors[ctx].append(token)\n    \n    context = max(context_scores, key=context_scores.get) if max(context_scores.values()) > 0 else \"general\"\n    \n    print(f\"CONTEXT DETECTION:\")\n    for ctx, score in context_scores.items():\n        if score > 0:\n            print(f\"   {ctx}: {score} matches → {matched_anchors[ctx]}\")\n    print(f\"   → Selected context: '{context}'\\n\")\n    \n    # Translate tokens\n    translated_sentence = []\n    for t in tokens:\n        # Skip line numbers\n        if re.match(r'^\\d+\\'?$', t):\n            print(f\"  Skipping line number: '{t}'\")\n            continue\n        \n        word_trans = translate_token_with_algolia(t, context, \"noun\", verbose=True, confidence_threshold=5)\n        translated_sentence.append(word_trans)\n    \n    final_translation = \" \".join(translated_sentence)\n    print(f\"\\n{'─'*40}\")\n    print(f\"FINAL TRANSLATION:\")\n    print(f\"   {final_translation}\")\n    print(f\"{'─'*40}\")\n    \n    results.append({\n        \"id\": row['id'],\n        \"translation\": final_translation\n    })\n    \n    # Stop after first 3 sentences for debugging (REMOVE FOR FULL RUN)\n    if row_idx >= 2:\n        print(f\"\\nSTOPPING AFTER 3 SENTENCES FOR DEBUGGING\")\n        print(f\"   Remove the 'if row_idx >= 2: break' line for full translation\")\n        break\n\n# Save for Kaggle Competition submission\nsubmission = pd.DataFrame(results)\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"SUBMISSION COMPLETE\")\nprint(\"=\"*80)\nprint(f\"Translated {len(results)} sentences\")\nprint(f\"Saved to submission.csv\")\nprint(\"=\"*80 + \"\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T06:08:50.106471Z","iopub.execute_input":"2026-01-24T06:08:50.106832Z","iopub.status.idle":"2026-01-24T06:09:23.127213Z","shell.execute_reply.started":"2026-01-24T06:08:50.106799Z","shell.execute_reply":"2026-01-24T06:09:23.125607Z"}},"outputs":[],"execution_count":null}]}