{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15061024},{"sourceType":"datasetVersion","sourceId":14221180,"datasetId":9071756,"databundleVersionId":15017441},{"sourceType":"datasetVersion","sourceId":14264475,"datasetId":8885793,"databundleVersionId":15064816},{"sourceType":"modelInstanceVersion","sourceId":692119,"databundleVersionId":15018534,"modelInstanceId":524756}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip uninstall tensorflow -y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:13:25.472300Z","iopub.execute_input":"2025-12-23T06:13:25.473058Z","iopub.status.idle":"2025-12-23T06:13:50.667294Z","shell.execute_reply.started":"2025-12-23T06:13:25.473012Z","shell.execute_reply":"2025-12-23T06:13:50.666589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-deps --no-index /kaggle/input/wheelf/sacremoses-0.1.1-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:13:50.669082Z","iopub.execute_input":"2025-12-23T06:13:50.669329Z","iopub.status.idle":"2025-12-23T06:13:52.278698Z","shell.execute_reply.started":"2025-12-23T06:13:50.669304Z","shell.execute_reply":"2025-12-23T06:13:52.277767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom transformers import MarianMTModel, MarianTokenizer\nfrom tqdm import tqdm\nimport os\nimport re\nimport sys","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:13:52.279995Z","iopub.execute_input":"2025-12-23T06:13:52.280259Z","iopub.status.idle":"2025-12-23T06:14:14.362772Z","shell.execute_reply.started":"2025-12-23T06:13:52.280233Z","shell.execute_reply":"2025-12-23T06:14:14.361825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/input/akkadian-model/pytorch/default/1\"\nTEST_FILE = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nPN_FILE = \"/kaggle/input/akkadian-translation/proper_nouns.csv\"\nSUBMISSION_FILE = \"submission.csv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.363916Z","iopub.execute_input":"2025-12-23T06:14:14.364677Z","iopub.status.idle":"2025-12-23T06:14:14.369062Z","shell.execute_reply.started":"2025-12-23T06:14:14.364640Z","shell.execute_reply":"2025-12-23T06:14:14.368101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    # Normalize Gaps\n    text = re.sub(r'\\[x\\]', ' <gap> ', text)\n    text = re.sub(r'…|\\[…\\s?…\\]|\\[\\.\\.\\.\\]', ' <big_gap> ', text)\n    # Remove uncertain markers (! ? / ˹ ˺)\n    text = re.sub(r'[!?:/˹˺]', '', text)\n    # Clean Brackets: [text] -> text\n    text = re.sub(r'\\[([^\\]]*)\\]', r'\\1', text)\n    # Normalize Determinatives: (d) -> {d}, (ki) -> {ki}\n    text = re.sub(r'\\(([a-zA-Z0-9]+)\\)', r'{\\1}', text)\n    \n    # Normalize Subscripts: il5 -> il₅\n    subscripts = str.maketrans(\"0123456789\", \"₀₁₂₃₄₅₆₇₈₉\")\n    def sub_repl(m): return m.group(1) + m.group(2).translate(subscripts)\n    text = re.sub(r'([a-zA-Z])([0-9])', sub_repl, text)\n    \n    # Collapse spaces\n    return re.sub(r'\\s+', ' ', text).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.371124Z","iopub.execute_input":"2025-12-23T06:14:14.371363Z","iopub.status.idle":"2025-12-23T06:14:14.387930Z","shell.execute_reply.started":"2025-12-23T06:14:14.371342Z","shell.execute_reply":"2025-12-23T06:14:14.387049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def translate_with_optimal_beams(texts, model, tokenizer, device):\n    \"\"\"Test different beam configurations\"\"\"\n    \n    configs = [\n        {'beams': 8, 'length_penalty': 0.8, 'no_repeat_ngram': 3},\n        {'beams': 10, 'length_penalty': 1.0, 'no_repeat_ngram': 3},\n        {'beams': 12, 'length_penalty': 1.2, 'no_repeat_ngram': 2},\n    ]\n    \n    all_translations = []\n    \n    for config in configs:\n        batch_translations = []\n        for i in range(0, len(texts), 32):\n            batch = texts[i:i+32]\n            encoded = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=128).to(device)\n            \n            with torch.no_grad():\n                generated = model.generate(\n                    **encoded,\n                    max_length=128,\n                    num_beams=config['beams'],\n                    length_penalty=config['length_penalty'],\n                    no_repeat_ngram_size=config['no_repeat_ngram'],\n                    early_stopping=True\n                )\n            \n            decoded = tokenizer.batch_decode(generated, skip_special_tokens=True)\n            batch_translations.extend(decoded)\n        \n        all_translations.append(batch_translations)\n    \n    # Vote or pick best\n    return all_translations[1]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.389008Z","iopub.execute_input":"2025-12-23T06:14:14.389599Z","iopub.status.idle":"2025-12-23T06:14:14.404963Z","shell.execute_reply.started":"2025-12-23T06:14:14.389536Z","shell.execute_reply":"2025-12-23T06:14:14.404180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def enhanced_postprocess(translation, source_text, proper_nouns):\n    \"\"\"Improved post-processing\"\"\"\n    \n    # 1. Capitalize first letter\n    if translation and len(translation) > 0:\n        translation = translation[0].upper() + translation[1:]\n    \n    # 2. Fix proper nouns from source\n    source_caps = re.findall(r'\\b[A-Z][a-z]+\\b', source_text)\n    for proper in source_caps:\n        # Case-insensitive replace\n        pattern = re.compile(re.escape(proper), re.IGNORECASE)\n        translation = pattern.sub(proper, translation)\n    \n    # 3. Apply known proper nouns\n    words = translation.split()\n    processed_words = []\n    for i, w in enumerate(words):\n        clean_w = w.strip(\".,!?\\\"'\").title()\n        if clean_w in proper_nouns or (i == 0 and len(w) > 0):\n            # Keep capitalization for proper nouns and first word\n            if i == 0:\n                processed_words.append(w[0].upper() + w[1:] if len(w) > 1 else w.upper())\n            else:\n                processed_words.append(w.title())\n        else:\n            processed_words.append(w)\n    translation = \" \".join(processed_words)\n    \n    # 4. Clean spacing\n    translation = re.sub(r'\\s+', ' ', translation).strip()\n    translation = re.sub(r'\\s+([.,!?])', r'\\1', translation)  # Remove space before punctuation\n    translation = re.sub(r'([.,!?])([A-Za-z])', r'\\1 \\2', translation)  # Add space after punctuation\n    \n    # 5. Ensure ends with punctuation\n    if translation and translation[-1] not in '.!?':\n        translation += '.'\n    \n    return translation","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.405986Z","iopub.execute_input":"2025-12-23T06:14:14.406386Z","iopub.status.idle":"2025-12-23T06:14:14.423169Z","shell.execute_reply.started":"2025-12-23T06:14:14.406351Z","shell.execute_reply":"2025-12-23T06:14:14.422484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_submission():\n    # 1. Load Resources\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    print(f\"Loading model on {device}...\")\n    \n    if not os.path.exists(MODEL_PATH):\n        print(f\"CRITICAL ERROR: Model path {MODEL_PATH} not found.\")\n        pd.DataFrame({'id': [], 'translation': []}).to_csv(SUBMISSION_FILE, index=False)\n        return\n\n    try:\n        model = MarianMTModel.from_pretrained(MODEL_PATH).to(device)\n        tokenizer = MarianTokenizer.from_pretrained(MODEL_PATH)\n        model.eval()  # Set to eval mode\n    except Exception as e:\n        print(f\"Error loading model: {e}\")\n        return\n\n    # 2. Load test data\n    print(\"Loading and cleaning test data...\")\n    try:\n        test_df = pd.read_csv(TEST_FILE)\n    except FileNotFoundError:\n        test_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\n    test_df['clean_transliteration'] = test_df['transliteration'].apply(clean_akkadian)\n    \n    # 3. Load proper nouns\n    proper_nouns = set()\n    if os.path.exists(PN_FILE):\n        pn_df = pd.read_csv(PN_FILE)\n        proper_nouns = set(pn_df['norm'].dropna().apply(lambda x: str(x).title()).tolist())\n        print(f\"Loaded {len(proper_nouns)} proper nouns\")\n\n    # 4. Generate translations with optimal beams\n    # **THIS IS THE KEY CHANGE - REPLACE OLD LOOP**\n    print(f\"Generating translations for {len(test_df)} items...\")\n    inputs = test_df['clean_transliteration'].tolist()\n    \n    # Use the new function instead of the old loop\n    translations = translate_with_optimal_beams(inputs, model, tokenizer, device)\n\n    # 5. Enhanced post-processing\n    print(\"Applying enhanced post-processing...\")\n    final_output = []\n    for i, text in enumerate(translations):\n        source = test_df.iloc[i]['clean_transliteration']\n        processed = enhanced_postprocess(text, source, proper_nouns)\n        final_output.append(processed)\n\n    # 6. Save submission\n    if len(final_output) != len(test_df):\n        print(f\"WARNING: Mismatch in lengths. Input: {len(test_df)}, Output: {len(final_output)}\")\n        while len(final_output) < len(test_df):\n            final_output.append(\"\")\n            \n    submission = pd.DataFrame({'id': test_df['id'], 'translation': final_output})\n    submission.to_csv(SUBMISSION_FILE, index=False)\n    print(f\"✓ Submission saved to {SUBMISSION_FILE} with {len(submission)} rows.\")\n    print(f\"✓ Used optimized beam search with 3 configurations\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.424123Z","iopub.execute_input":"2025-12-23T06:14:14.424468Z","iopub.status.idle":"2025-12-23T06:14:14.444646Z","shell.execute_reply.started":"2025-12-23T06:14:14.424431Z","shell.execute_reply":"2025-12-23T06:14:14.443888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"generate_submission()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:14.445689Z","iopub.execute_input":"2025-12-23T06:14:14.446608Z","iopub.status.idle":"2025-12-23T06:14:24.837351Z","shell.execute_reply.started":"2025-12-23T06:14:14.446572Z","shell.execute_reply":"2025-12-23T06:14:24.836465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"check_csv = pd.read_csv('/kaggle/working/submission.csv')\ncheck_csv.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T06:14:24.838315Z","iopub.execute_input":"2025-12-23T06:14:24.838664Z","iopub.status.idle":"2025-12-23T06:14:24.858481Z","shell.execute_reply.started":"2025-12-23T06:14:24.838629Z","shell.execute_reply":"2025-12-23T06:14:24.857720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}