{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":698224,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":529626,"modelId":543620}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport logging\nfrom transformers import logging as hf_logging\n\nhf_logging.set_verbosity_error()\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:19.761565Z","iopub.execute_input":"2026-01-09T11:37:19.762124Z","iopub.status.idle":"2026-01-09T11:37:25.351601Z","shell.execute_reply.started":"2026-01-09T11:37:19.762098Z","shell.execute_reply":"2026-01-09T11:37:25.350984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport glob\nimport pandas as pd\nimport numpy as np\nimport torch\nimport warnings\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import (\n    Trainer, \n    TrainingArguments, \n    DataCollatorWithPadding,\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer, \n    EarlyStoppingCallback\n)\nfrom sklearn.model_selection import GroupKFold\nfrom difflib import SequenceMatcher\n\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:25.352808Z","iopub.execute_input":"2026-01-09T11:37:25.353126Z","iopub.status.idle":"2026-01-09T11:37:48.636264Z","shell.execute_reply.started":"2026-01-09T11:37:25.353103Z","shell.execute_reply":"2026-01-09T11:37:48.635485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    DATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation/\"\n    OUTPUT_DIR = \"./results\"\n    \n    MODEL_PATH = \"/kaggle/input/nllb-200-distilled-600m\"\n    \n    if not os.path.exists(os.path.join(MODEL_PATH, \"config.json\")):\n        potential_paths = glob.glob(f\"{MODEL_PATH}/**/config.json\", recursive=True)\n        if potential_paths:\n            MODEL_PATH = os.path.dirname(potential_paths[0])\n            print(f\"✅ Found correct model sub-folder at: {MODEL_PATH}\")\n    else:\n        print(f\"✅ Found NLLB Model at: {MODEL_PATH}\")\n    # HYPERPARAMETERS\n    MAX_LEN = 75\n    BATCH_SIZE = 2       \n    GRAD_ACCUM = 8       \n    LEARNING_RATE = 2e-5 \n    EPOCHS = 15     \n    FOLDS = 5            \n    SEED = 42\n    OUTPUT_DIR = \"./results\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:48.637140Z","iopub.execute_input":"2026-01-09T11:37:48.637761Z","iopub.status.idle":"2026-01-09T11:37:48.648923Z","shell.execute_reply.started":"2026-01-09T11:37:48.637737Z","shell.execute_reply":"2026-01-09T11:37:48.648182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ENHANCED MINING ENGINE (PDF + DICT + LEXICON)\ndef mine_enhanced_data():\n    mined_data = []\n    \n    # Dictionary & Lexicon\n    print(\"\\n📚 [MINER] Extracting Dictionary/Lexicon vocabulary...\")\n    try:\n        dict_df = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"eBL_Dictionary.csv\"))\n        lex_df = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"OA_Lexicon_eBL.csv\"))\n        \n        # Word-level mapping from Dictionary\n        for _, row in dict_df.iterrows():\n            if pd.notna(row['lemma']) and pd.notna(row['meaning']):\n                mined_data.append({'transliteration': str(row['lemma']), 'translation': str(row['meaning']), 'source': 'dict'})\n        \n        # Word-level mapping from Lexicon\n        for _, row in lex_df.iterrows():\n            if pd.notna(row['transliteration']) and pd.notna(row['translation']):\n                mined_data.append({'transliteration': str(row['transliteration']), 'translation': str(row['translation']), 'source': 'lex'})\n    except Exception as e:\n        print(f\"⚠️ Dictionary/Lexicon mining issue: {e}\")\n\n    # PDF Scanning\n    print(\"⛏️ [MINER] Scanning PDFs for hidden tablet translations...\")\n    try:\n        pub_texts = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"published_texts.csv\"))\n        publications = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"publications.csv\"))\n        pdf_map = dict(zip(publications['pdf_name'], publications['page_text']))\n        \n        targets = pub_texts.dropna(subset=['publication_catalog'])\n        for idx, row in targets.iterrows():\n            cat_id, translit = str(row['publication_catalog']), row['transliteration']\n            for pdf_name, text in pdf_map.items():\n                text = str(text)\n                if cat_id in text:\n                    start = text.find(cat_id)\n                    candidate = re.sub(r'[^A-Za-z0-9\\s.,]', '', text[start:start+500])\n                    candidate = re.sub(r'\\s+', ' ', candidate).strip()\n                    if len(candidate) > 20 and (\"the\" in candidate.lower() or \"and\" in candidate.lower()):\n                        mined_data.append({'transliteration': translit, 'translation': candidate, 'source': 'pdf'})\n                        break\n    except Exception as e:\n        print(f\"⚠️ PDF mining issue: {e}\")\n\n    print(f\"✅ [MINER] Success! Extracted {len(mined_data)} extra training items.\")\n    return pd.DataFrame(mined_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:48.650673Z","iopub.execute_input":"2026-01-09T11:37:48.650913Z","iopub.status.idle":"2026-01-09T11:37:48.958317Z","shell.execute_reply.started":"2026-01-09T11:37:48.650893Z","shell.execute_reply":"2026-01-09T11:37:48.957551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CLEANING & NORMALIZATION\ndef clean_text(text):\n    if not isinstance(text, str): return \"\"\n    # remove punctuation, not the sounds of the language\n    text = re.sub(r'[!\\?/:.˺˹\\[\\]]', '', text) \n    text = text.replace('[x]', '<gap>').replace('...', '<big_gap>')\n    return re.sub(r'\\s+', ' ', text).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:48.959201Z","iopub.execute_input":"2026-01-09T11:37:48.959472Z","iopub.status.idle":"2026-01-09T11:37:48.972923Z","shell.execute_reply.started":"2026-01-09T11:37:48.959446Z","shell.execute_reply":"2026-01-09T11:37:48.972420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_models():\n    # LOAD DATA\n    print(\"📂 [PREP] Loading data...\")\n    train_df = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"train.csv\"))\n    \n    # RUN MINER & CLEAN\n    extra_df = mine_enhanced_data()\n    if not extra_df.empty:\n        train_df = pd.concat([train_df, extra_df], ignore_index=True)\n\n    # DEDUPLICATION (Stop the model from memorizing duplicates)\n    train_df = train_df.drop_duplicates(subset=['transliteration', 'translation'])\n    \n    print(\"🧹 [PREP] Cleaning text...\")\n    train_df['input_text'] = train_df['transliteration'].apply(clean_text)\n    train_df['target_text'] = train_df['translation'].apply(clean_text)\n\n     # Remove \"Long Junk\" that confuses the model\n    train_df = train_df[train_df['target_text'].str.split().str.len() < 50]\n    \n    # SETUP TOKENIZER & FOLDS\n    tokenizer = AutoTokenizer.from_pretrained(CONFIG.MODEL_PATH)\n    gkf = GroupKFold(n_splits=CONFIG.FOLDS)\n    train_df['group_id'] = train_df['oare_id'].fillna(train_df.index.to_series()).astype(str)\n    \n    saved_model_paths = []\n    \n    # TRAINING LOOP\n    for fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups=train_df['group_id'])):\n        print(f\"\\n🚀 [FOLD {fold+1}/{CONFIG.FOLDS}] Training Started...\")\n        train_sub, val_sub = train_df.iloc[train_idx], train_df.iloc[val_idx]\n        \n        model = AutoModelForSeq2SeqLM.from_pretrained(CONFIG.MODEL_PATH)\n\n        # TOKENIZATION LOGIC\n        def prepare_data(df):\n            tokenizer.src_lang = \"akk_Latn\"\n            tokenizer.tgt_lang = \"eng_Latn\"\n            \n            # Tokenize Inputs\n            model_inputs = tokenizer(df['input_text'].tolist(), max_length=CONFIG.MAX_LEN, \n                                     truncation=True, padding=\"max_length\")\n            \n            # Tokenize Targets\n            with tokenizer.as_target_tokenizer():\n                labels = tokenizer(df['target_text'].tolist(), max_length=CONFIG.MAX_LEN, \n                                   truncation=True, padding=\"max_length\")\n            \n            final_labels = []\n            for label_set in labels[\"input_ids\"]:\n                new_labels = [(l if l != tokenizer.pad_token_id else -100) for l in label_set]\n                final_labels.append(new_labels)\n                \n            model_inputs[\"labels\"] = final_labels\n            return model_inputs\n\n        train_data = prepare_data(train_sub)\n        val_data = prepare_data(val_sub)\n\n        class SimpleDataset(Dataset):\n            def __init__(self, data): self.data = data\n            def __len__(self): return len(self.data[\"input_ids\"])\n            def __getitem__(self, i): return {k: torch.tensor(v[i]) for k, v in self.data.items()}\n\n        # TRAINING ARGS\n        args = TrainingArguments(\n            output_dir=f\"{CONFIG.OUTPUT_DIR}/fold{fold}\",\n            eval_strategy=\"epoch\",\n            save_strategy=\"no\",\n            learning_rate=CONFIG.LEARNING_RATE,\n            per_device_train_batch_size=CONFIG.BATCH_SIZE,\n            gradient_accumulation_steps=CONFIG.GRAD_ACCUM,\n            num_train_epochs=CONFIG.EPOCHS,\n            weight_decay=0.1,\n            save_total_limit=1,\n            load_best_model_at_end=False,\n            metric_for_best_model=\"loss\",\n            fp16=torch.cuda.is_available(),\n            remove_unused_columns=False,\n            report_to=\"none\"\n        )\n\n        # TRAINER\n        trainer = Trainer(\n            model=model,\n            args=args,\n            train_dataset=SimpleDataset(train_data),\n            eval_dataset=SimpleDataset(val_data),\n            data_collator=DataCollatorWithPadding(tokenizer=tokenizer)\n        )\n\n        trainer.train()\n        \n        # 8. SAVE & CLEANUP\n        save_path = f\"./models/fold{fold}\"\n        os.makedirs(save_path, exist_ok=True)\n        model.save_pretrained(save_path)\n        tokenizer.save_pretrained(save_path)\n        saved_model_paths.append(save_path)\n\n        # GPU RAM Management\n        del model, trainer\n        torch.cuda.empty_cache()\n        gc.collect()\n\n    return saved_model_paths, train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:48.974009Z","iopub.execute_input":"2026-01-09T11:37:48.974270Z","iopub.status.idle":"2026-01-09T11:37:48.995066Z","shell.execute_reply.started":"2026-01-09T11:37:48.974239Z","shell.execute_reply":"2026-01-09T11:37:48.994418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# HYBRID INFERENCE (SEARCH + AI)\ndef hybrid_predict(model_paths, memory_df):\n    print(\"\\n🔮 [HYBRID] Starting Inference...\")\n    test_df = pd.read_csv(os.path.join(CONFIG.DATA_DIR, \"test.csv\"))\n\n    # Filter Memory Map\n    memory_df = memory_df[~memory_df['translation'].str.contains(\"Kanesh|dagger|testimony\", case=False)]\n    memory_map = dict(zip(memory_df['input_text'], memory_df['translation']))\n    \n    # Load BEST Model\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    model = AutoModelForSeq2SeqLM.from_pretrained(model_paths[0]).to(device)\n    tokenizer = AutoTokenizer.from_pretrained(model_paths[0])\n    \n    final_preds = []\n    \n    # Prediction Loop\n    for idx, row in test_df.iterrows():\n        clean_in = clean_text(row['transliteration'])\n        \n        # EXACT SEARCH\n        if clean_in in memory_map:\n            final_preds.append(memory_map[clean_in])\n            continue\n            \n        # AI PREDICTION (NLLB)\n        inputs = tokenizer(clean_in, return_tensors=\"pt\", padding=True).to(device)\n        forced_bos = tokenizer.convert_tokens_to_ids(\"eng_Latn\")\n        \n        with torch.no_grad():\n            gen = model.generate(\n                **inputs, \n                forced_bos_token_id=forced_bos, \n                max_length=CONFIG.MAX_LEN,           \n                num_beams=2,            \n                repetition_penalty=3.5,  \n                length_penalty=1.0\n            )\n            \n        pred = tokenizer.decode(gen[0], skip_special_tokens=True)\n        final_preds.append(pred)\n    \n    # Save\n    sub = pd.DataFrame({'id': test_df['id'], 'translation': final_preds})\n    sub.to_csv('submission.csv', index=False)\n    print(\"✅ submission.csv saved successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:48.995929Z","iopub.execute_input":"2026-01-09T11:37:48.996226Z","iopub.status.idle":"2026-01-09T11:37:49.015335Z","shell.execute_reply.started":"2026-01-09T11:37:48.996162Z","shell.execute_reply":"2026-01-09T11:37:49.014794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# RUNNER\nif __name__ == \"__main__\":\n    if torch.cuda.is_available():\n        print(\"🚀 GPU Detected. Starting Pipeline...\")\n        # Train models and get data for memory\n        paths, memory_data = train_models()\n        # Run Hybrid Prediction\n        hybrid_predict(paths, memory_data)\n    else:\n        print(\"❌ Error: No GPU. Go to Settings -> Accelerator -> GPU P100.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T11:37:49.016121Z","iopub.execute_input":"2026-01-09T11:37:49.016392Z","iopub.status.idle":"2026-01-09T12:39:51.880731Z","shell.execute_reply.started":"2026-01-09T11:37:49.016367Z","shell.execute_reply":"2026-01-09T12:39:51.879928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = \"./models/fold0\" \ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# Load Tokenizer & Model\ntokenizer = AutoTokenizer.from_pretrained(model_path, src_lang=\"akk_Latn\")\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_path).to(device)\nmodel.eval() \n\ntest_sentence = \"awīlum damiq\"\n\n# Tokenize with clear Target Language\ninputs = tokenizer(test_sentence, return_tensors=\"pt\").to(device)\nforced_bos = tokenizer.convert_tokens_to_ids(\"eng_Latn\")\n\n# Optimized Generation Settings\nwith torch.no_grad():\n    outputs = model.generate(\n        **inputs,\n        forced_bos_token_id=forced_bos, \n        \n        # STOP THE HALLUCINATION \n        max_new_tokens=25,      \n        num_beams=2,             \n        repetition_penalty=3.5,  \n        length_penalty=0.8,      \n        no_repeat_ngram_size=2,  \n        early_stopping=True\n    )\n\ntranslation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n\nprint(f\"\\n📜 Akkadian: {test_sentence}\")\nprint(f\"🌍 English AI: {translation}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T12:45:29.888150Z","iopub.execute_input":"2026-01-09T12:45:29.888802Z","iopub.status.idle":"2026-01-09T12:45:32.378316Z","shell.execute_reply.started":"2026-01-09T12:45:29.888769Z","shell.execute_reply":"2026-01-09T12:45:32.377574Z"}},"outputs":[],"execution_count":null}]}