{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14266463,"sourceType":"datasetVersion","datasetId":9103868}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport os\nimport sys\nimport subprocess\nimport socket\n\n# ==========================================\n# CRITICAL FIXES (Run before imports)\n# ==========================================\n# 1. Fix 'MessageFactory' / Protobuf error (Offline Fix)\nos.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'\n\n# 2. Suppress TensorFlow/Linkage logs\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'\n# FORCE SINGLE GPU to avoid DataParallel OOM overhead on T4s\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n\nimport numpy as np\nimport pandas as pd\nimport re\nimport random\nimport torch\nimport glob\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer, \n    DataCollatorForSeq2Seq,\n    NllbTokenizer\n)\nfrom datasets import Dataset\n\n# ==========================================\n# Configuration & Offline Setup\n# ==========================================\nINPUT_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\nWORKING_DIR = \"/kaggle/working\"\nMAX_LENGTH = 128\n\n# ---------------------------------------------------------\n# AUTO-DETECT MODEL PATH (Offline Mode)\n# ---------------------------------------------------------\ndef find_local_model():\n    print(\"🔍 Scanning /kaggle/input for models...\")\n    \n    # 1. Direct check of the path the user explicitly mentioned\n    explicit_path = \"/kaggle/input/nllb-200-distilled-600m\"\n    if os.path.exists(explicit_path):\n        print(f\"   Checking explicit path: {explicit_path}\")\n        # List contents for debug\n        try:\n            print(f\"   Contents: {os.listdir(explicit_path)}\")\n        except:\n            pass\n        \n        # Check if this IS the model dir (flat structure)\n        if os.path.exists(os.path.join(explicit_path, \"config.json\")):\n            print(f\"   ✅ Found config.json in {explicit_path}\")\n            return explicit_path\n\n    # 2. Deep Scan using os.walk (more robust than glob)\n    print(\"   Performing deep scan of /kaggle/input...\")\n    for root, dirs, files in os.walk(\"/kaggle/input\"):\n        if \"config.json\" in files:\n            # We found a potential model directory\n            print(f\"   ❓ Candidate: {root}\")\n            \n            # Filter out the competition data itself\n            if \"deep-past-initiative\" in root:\n                print(\"      Skipping (looks like competition data)\")\n                continue\n                \n            # Relaxed Check: Needs config.json AND (tokenizer.json OR tokenizer_config.json OR sentencepiece model)\n            has_tokenizer = \"tokenizer.json\" in files or \\\n                            \"tokenizer_config.json\" in files or \\\n                            \"sentencepiece.bpe.model\" in files or \\\n                            \"spiece.model\" in files\n                            \n            if has_tokenizer:\n                print(f\"   ✅ VALID MODEL FOUND at: {root}\")\n                return root\n            else:\n                print(f\"      ❌ Missing tokenizer files. Contents: {files}\")\n\n    return None\n\nMODEL_CHECKPOINT = find_local_model()\n\nif MODEL_CHECKPOINT:\n    print(f\"✅ FINAL MODEL PATH: {MODEL_CHECKPOINT}\")\nelse:\n    print(\"\\n\" + \"!\"*60)\n    print(\"❌ FATAL ERROR: MODEL NOT FOUND LOCALLY\")\n    print(\"!\"*60)\n    print(\"Debug Info: The script scanned /kaggle/input but could not find a valid model folder.\")\n    print(\"A valid folder must contain 'config.json' AND a tokenizer file (tokenizer.json or sentencepiece.bpe.model).\")\n    print(\"!\"*60 + \"\\n\")\n    # STOP EXECUTION HERE\n    sys.exit(\"Stopping execution: Model missing.\")\n\nprint(\"-\" * 30)\nif torch.cuda.is_available():\n    DEVICE = \"cuda\"\n    print(f\"✅ GPU DETECTED: {torch.cuda.get_device_name(0)}\")\nelse:\n    DEVICE = \"cpu\"\n    print(\"❌ ERROR: GPU NOT DETECTED.\")\n    print(\"Training will be extremely slow.\")\n    print(\"FIX: Go to Settings -> Accelerator -> Select 'GPU T4 x2'.\")\nprint(\"-\" * 30)\n\n# ==========================================\n# Helper Functions\n# ==========================================\n\ndef load_lexicon(lexicon_path):\n    if not os.path.exists(lexicon_path):\n        return {}\n    lex = pd.read_csv(lexicon_path)\n    return {str(r['form']).strip(): str(r['norm']).strip() \n            for _, r in lex.iterrows() \n            if pd.notna(r['form']) and pd.notna(r['norm'])} # Corrected: Removed unnecessary quotes around r['form'] and r['norm']\n\ndef clean_text(text):\n    if not isinstance(text, str): return \"\"\n    text = re.sub(r'[!?/:,]', '', text)\n    text = text.replace('{d}', '[GOD]').replace('(d)', '[GOD]')\n    text = text.replace('{ki}', '[GEO]').replace('(ki)', '[GEO]')\n    return re.sub(r'\\s+', ' ', text).strip() # Corrected: Escaped backslash in regex\n\ndef lexicon_augment(text, form_to_norm, prob=0.3):\n    if not form_to_norm or random.random() > prob:\n        return text\n    return \" \".join([form_to_norm.get(w, w) if random.random() < 0.5 else w for w in text.split()])\n\n# ==========================================\n# Main Pipeline\n# ==========================================\n\ndef main():\n    # 0. Debug Version & Clear Cache\n    import transformers\n    import gc\n    torch.cuda.empty_cache()\n    gc.collect()\n    print(f\"DEBUG: Transformers Version: {transformers.__version__}\")\n\n    # 1. Load Data\n    print(\"Loading Data...\")\n    train_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n    test_df = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))\n    lexicon_path = os.path.join(INPUT_DIR, 'OA_Lexicon_eBL.csv')\n    form_to_norm = load_lexicon(lexicon_path)\n    \n    # 2. Preprocess\n    print(\"Preprocessing...\")\n    train_df['cleaned'] = train_df['transliteration'].apply(clean_text)\n    test_df['cleaned'] = test_df['transliteration'].apply(clean_text)\n    test_df['norm'] = test_df['cleaned'].apply(lambda x: \" \".join([form_to_norm.get(w, w) for w in x.split()]))\n    \n    # 3. Model Loading\n    print(f\"Loading Tokenizer from: {MODEL_CHECKPOINT}...\")\n    try:\n        # First try standard AutoTokenizer\n        tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, local_files_only=True)\n    except Exception as e:\n        print(f\"⚠️ AutoTokenizer failed ({e}). Attempting fallback to NllbTokenizer...\")\n        try:\n            # Fallback: Point directly to the sentencepiece model\n            spm_path = os.path.join(MODEL_CHECKPOINT, \"sentencepiece.bpe.model\")\n            if not os.path.exists(spm_path):\n                raise FileNotFoundError(f\"Fallback failed: {spm_path} not found.\")\n            \n            tokenizer = NllbTokenizer(vocab_file=spm_path, src_lang=\"eng_Latn\", tgt_lang=\"eng_Latn\")\n            print(\"✅ Fallback to NllbTokenizer successful.\")\n        except Exception as e_inner:\n            print(f\"❌ CRITICAL: Could not load tokenizer. Error: {e_inner}\")\n            return\n\n    tokenizer.add_special_tokens({'additional_special_tokens': ['[GOD]', '[GEO]']})\n    tokenizer.src_lang = \"eng_Latn\"\n    tokenizer.tgt_lang = \"eng_Latn\"\n    \n    # 4. Dataset Creation\n    def preprocess_fn(batch):\n        inputs = [lexicon_augment(ex, form_to_norm) for ex in batch[\"cleaned\"]]\n        model_inputs = tokenizer(inputs, max_length=MAX_LENGTH, truncation=True, padding=\"max_length\")\n        labels = tokenizer(text_target=batch[\"translation\"], max_length=MAX_LENGTH, truncation=True, padding=\"max_length\")\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    dataset = Dataset.from_pandas(train_df).train_test_split(test_size=0.1, seed=42)\n    tokenized_ds = dataset.map(preprocess_fn, batched=True, remove_columns=dataset[\"train\"].column_names)\n    \n    # 5. Training Setup\n    print(\"Initializing Model...\")\n    model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CHECKPOINT, local_files_only=True)\n    model.resize_token_embeddings(len(tokenizer))\n    # Note: We do NOT call model.to(DEVICE) here manually. Trainer handles it.\n    \n    from transformers import GenerationConfig\n    \n    # Define generation config for better inference quality (fixes repetition)\n    gen_config = GenerationConfig(\n        max_length=MAX_LENGTH,\n        num_beams=5,\n        repetition_penalty=1.2,\n        no_repeat_ngram_size=3,\n        early_stopping=True,\n        decoder_start_token_id=tokenizer.lang_code_to_id[tokenizer.tgt_lang] if hasattr(tokenizer, 'lang_code_to_id') else 2\n    )\n\n    training_args = Seq2SeqTrainingArguments(\n        output_dir=\"./results\",\n        eval_strategy=\"epoch\",\n        save_strategy=\"epoch\", \n        learning_rate=4e-5, \n        per_device_train_batch_size=4 if DEVICE == \"cuda\" else 4,\n        gradient_accumulation_steps=4,\n        save_total_limit=1, \n        save_only_model=True, # Back to True to save disk (we will handle loading manually)\n        num_train_epochs=10, \n        predict_with_generate=True,\n        generation_config=gen_config, \n        fp16=(DEVICE == \"cuda\"),\n        gradient_checkpointing=False, \n        logging_steps=50,\n        report_to=\"none\",\n        load_best_model_at_end=False, # DISABLED to prevent \"missing keys\" error on reload\n        metric_for_best_model=\"eval_loss\",\n        greater_is_better=False,\n        weight_decay=0.02, \n        warmup_ratio=0.1, \n    )\n    \n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=tokenized_ds[\"train\"],\n        eval_dataset=tokenized_ds[\"test\"],\n        tokenizer=tokenizer,\n        data_collator=DataCollatorForSeq2Seq(tokenizer, model=model),\n    )\n    \n    # 6. Train\n    print(\"Starting Training...\")\n    trainer.train()\n    \n    # MANUAL BEST MODEL LOADING\n    # Find the latest checkpoint (since we limit to 1, it should be the best/last kept)\n    print(\"Loading best saved model from disk...\")\n    checkpoints = [d for d in os.listdir(\"./results\") if d.startswith(\"checkpoint\")]\n    if checkpoints:\n        # Sort by number to get the latest one\n        best_ckpt = sorted(checkpoints, key=lambda x: int(x.split('-')[1]))[-1]\n        best_model_path = os.path.join(\"./results\", best_ckpt)\n        print(f\"✅ Loading model from: {best_model_path}\")\n        \n        # Reload model weights manually\n        model = AutoModelForSeq2SeqLM.from_pretrained(best_model_path, local_files_only=True)\n        model.to(DEVICE)\n    else:\n        print(\"⚠️ No checkpoints found. Using current model state.\")\n    \n    # CRITICAL: Delete checkpoints to free disk space for inference/submission\n    print(\"Cleaning up checkpoints to save disk space...\")\n    import shutil\n    try:\n        shutil.rmtree(\"./results\")\n        print(\"✅ Checkpoints deleted.\")\n    except Exception as e:\n        print(f\"⚠️ Could not delete checkpoints: {e}\")\n    \n    # 7. Inference\n    print(f\"Starting Inference (Test Set Size: {len(test_df)})...\")\n    if len(test_df) < 10:\n        print(\"ℹ️ Note: Test set is small (placeholder). Inference will be instant.\")\n        \n    model.eval()\n    test_texts = test_df['norm'].tolist()\n    translations = []\n    \n    batch_size = 32 if DEVICE == \"cuda\" else 8\n    for i in range(0, len(test_texts), batch_size):\n        batch = test_texts[i : i + batch_size]\n        inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=MAX_LENGTH).to(DEVICE)\n        with torch.no_grad():\n            # Explicitly pass params to guarantee they are used\n            outputs = model.generate(\n                **inputs, \n                max_length=MAX_LENGTH,\n                num_beams=5,\n                repetition_penalty=1.2,\n                no_repeat_ngram_size=3,\n                early_stopping=True\n            )\n        translations.extend(tokenizer.batch_decode(outputs, skip_special_tokens=True))\n        if i % 100 == 0: print(f\"Progress: {i}/{len(test_texts)}\")\n\n    # 8. Save\n    pd.DataFrame({'id': test_df['id'], 'translation': translations}).to_csv('submission.csv', index=False)\n    print(\"✅ Done! submission.csv saved.\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-23T11:31:56.829312Z","iopub.execute_input":"2025-12-23T11:31:56.829708Z","iopub.status.idle":"2025-12-23T11:50:15.686042Z","shell.execute_reply.started":"2025-12-23T11:31:56.829680Z","shell.execute_reply":"2025-12-23T11:50:15.684886Z"}},"outputs":[],"execution_count":null}]}