""" Deep Past Winning Kernel - ByT5 Model Soup Based on jiexusheng20bz/byt-ensemble research Models: 1. jeanjean111/byt5-base-big-data2 (weight 0.97) 2. llkh0a/byt5-akkadian-model (weight 0.99) 3. qifeihhh666/byt5-base-akkadian-gap-sentence2 (weight 0.60) Expected: 35-38 BLEU """ import os import re import json import torch import numpy as np import pandas as pd from pathlib import Path from tqdm import tqdm from transformers import AutoTokenizer, AutoModelForSeq2SeqLM def preprocess_text(text): """Preprocess Akkadian text with gap markers.""" if pd.isna(text): return "" text = str(text).strip() # Convert ellipsis and unicode dots to text = re.sub(r'(\.{3,}|…+|……)', '', text) # Convert xx markers to text = re.sub(r'(xx+|\s+x\s+)', '', text) # Normalize whitespace text = re.sub(r'\s+', ' ', text) return text.strip() def postprocess_translation(text): """Clean up model output.""" if not text: return "" # Remove extra whitespace text = re.sub(r'\s+', ' ', text).strip() # Fix common fraction patterns text = re.sub(r'1\s*/\s*2', '1/2', text) text = re.sub(r'1\s*/\s*3', '1/3', text) text = re.sub(r'2\s*/\s*3', '2/3', text) text = re.sub(r'1\s*/\s*4', '1/4', text) text = re.sub(r'3\s*/\s*4', '3/4', text) # Remove repetition artifacts words = text.split() if len(words) > 3: # Check for repeated phrases for length in [3, 2]: if len(words) >= length * 2: first_part = ' '.join(words[:length]) second_part = ' '.join(words[length:length*2]) if first_part == second_part: words = words[length:] break text = ' '.join(words) # Ensure ASCII for English output text = text.encode('ascii', 'ignore').decode('ascii') return text.strip() def load_model_and_tokenizer(model_name, device): """Load a ByT5 model and tokenizer.""" print(f"Loading {model_name}...") tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForSeq2SeqLM.from_pretrained(model_name) model = model.to(device) model.eval() return model, tokenizer def generate_translation(model, tokenizer, text, device, num_beams=8, length_penalty=1.10): """Generate translation for a single text.""" inputs = tokenizer(text, return_tensors="pt", max_length=512, truncation=True) inputs = {k: v.to(device) for k, v in inputs.items()} with torch.no_grad(): outputs = model.generate( **inputs, max_length=256, num_beams=num_beams, length_penalty=length_penalty, early_stopping=True, no_repeat_ngram_size=3, ) translation = tokenizer.decode(outputs[0], skip_special_tokens=True) return translation def average_model_weights(models, weights): """Average model weights with given weights.""" # Normalize weights weights = np.array(weights, dtype=np.float32) weights = weights / weights.sum() print(f"Using normalized weights: {weights}") # Get state dicts state_dicts = [model.state_dict() for model in models] # Average parameters averaged_state = {} for key in state_dicts[0].keys(): averaged_state[key] = sum( w * sd[key] for w, sd in zip(weights, state_dicts) ) return averaged_state def main(): print("=" * 60) print("Deep Past - ByT5 Model Soup Inference") print("=" * 60) # Check device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}") # Model configurations MODEL_NAMES = [ "jeanjean111/byt5-base-big-data2", "llkh0a/byt5-akkadian-model", "qifeihhh666/byt5-base-akkadian-gap-sentence2" ] MODEL_WEIGHTS = [0.97, 0.99, 0.60] print(f"\nLoading {len(MODEL_NAMES)} models...") print(f"Raw weights: {MODEL_WEIGHTS}") # Load all models models = [] tokenizers = [] for model_name in MODEL_NAMES: try: model, tokenizer = load_model_and_tokenizer(model_name, device) models.append(model) tokenizers.append(tokenizer) print(f" Loaded: {model_name}") except Exception as e: print(f" ERROR loading {model_name}: {e}") raise print(f"\nSuccessfully loaded {len(models)} models") # Create soup by weight averaging print("\nCreating model soup with weight averaging...") averaged_state = average_model_weights(models, MODEL_WEIGHTS) # Load averaged weights into first model (we'll use its tokenizer) soup_model = models[0] soup_model.load_state_dict(averaged_state) soup_tokenizer = tokenizers[0] print("Model soup created successfully!") # Free memory from other models for i, model in enumerate(models[1:], 1): del model torch.cuda.empty_cache() # Load test data test_path = "/kaggle/input/deep-past-initiative-machine-translation/test.csv" print(f"\nLoading test data from {test_path}...") if not Path(test_path).exists(): raise FileNotFoundError(f"Test file not found: {test_path}") test_df = pd.read_csv(test_path) print(f"Loaded {len(test_df)} test samples") # Preprocess print("\nPreprocessing texts...") test_df['processed'] = test_df['akkadian'].apply(preprocess_text) # Generate translations print(f"\nGenerating translations (beams=8, length_penalty=1.10)...") translations = [] for idx, row in tqdm(test_df.iterrows(), total=len(test_df)): text = row['processed'] if not text: translations.append("") continue try: translation = generate_translation( soup_model, soup_tokenizer, text, device, num_beams=8, length_penalty=1.10 ) translation = postprocess_translation(translation) translations.append(translation) except Exception as e: print(f"\nError on sample {idx}: {e}") translations.append("") # Create submission submission = pd.DataFrame({ 'id': test_df['id'], 'translation': translations }) # Validate print(f"\nSubmission shape: {submission.shape}") print(f"Columns: {list(submission.columns)}") print(f"Non-empty translations: {sum(1 for t in translations if t)}") # Save output_path = "/kaggle/working/submission.csv" submission.to_csv(output_path, index=False) print(f"\nSaved submission to {output_path}") # Show samples print("\n" + "=" * 60) print("Sample outputs:") print("=" * 60) for i in range(min(5, len(submission))): print(f"\nID: {submission.iloc[i]['id']}") print(f"Input: {test_df.iloc[i]['akkadian'][:80]}...") print(f"Output: {submission.iloc[i]['translation']}") print("\n" + "=" * 60) print("Inference complete!") print("=" * 60) if __name__ == "__main__": main()