""" Deep Past - ByT5 Model Soup (Offline Version) Based on jiexusheng20bz/byt-ensemble (38+ BLEU) Models loaded from Kaggle datasets (no internet required): 1. jeanjean111/byt5-base-big-data2 (weight 0.97) 2. llkh0a/byt5-akkadian-model (weight 0.99) 3. qifeihhh666/train-gap-all-2 (weight 0.60) """ import os import re import pandas as pd import torch from torch.utils.data import DataLoader, Dataset from transformers import AutoTokenizer, AutoModelForSeq2SeqLM from tqdm.auto import tqdm # Configuration - EXACT copy from winning kernel CONFIG = { "data_path": "/kaggle/input/deep-past-initiative-machine-translation/test.csv", "models": [ "/kaggle/input/byt5-base-big-data2", "/kaggle/input/byt5-akkadian-model", "/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2" ], "model_weights": [0.97, 0.99, 0.60], "device": torch.device("cuda" if torch.cuda.is_available() else "cpu"), "max_len": 512, "batch_size": 8, "gen_params": { "num_beams": 8, "max_new_tokens": 512, "length_penalty": 1.10, "early_stopping": True, "no_repeat_ngram_size": 4, # CRITICAL: Prevents repeating 4+ word sequences "repetition_penalty": 1.5 # CRITICAL: Penalizes any repetition } } def preprocess_transliteration(text): """Preprocess Akkadian transliteration - EXACT copy from winning kernel.""" if pd.isna(text): return "" processed_text = str(text) processed_text = re.sub(r'(\.{3,}|…+|……)', '', processed_text) processed_text = re.sub(r'(xx+|\s+x\s+)', '', processed_text) return processed_text def postprocess_translation(text): """Clean up model output - EXACT copy from winning kernel.""" if not isinstance(text, str) or not text.strip(): return "" processed_text = text.replace('ḫ', 'h').replace('Ḫ', 'H') sub_map = str.maketrans("₀₁₂₃₄₅₆₇₈₉", "0123456789") processed_text = processed_text.translate(sub_map) processed_text = re.sub(r'(\[x\]|\(x\)|\bx\b)', '', processed_text, flags=re.I) processed_text = re.sub(r'(\.{3,}|…|\[\.+\])', '', processed_text) processed_text = re.sub(r'\s*', ' ', processed_text) processed_text = re.sub(r'\s*', ' ', processed_text) processed_text = re.sub(r'\((fem|plur|pl|sing|singular|plural|\?|!)\.?\s*\w*\)', '', processed_text, flags=re.I) processed_text = processed_text.replace('', '\x00GAP\x00').replace('', '\x00BIG\x00') # Remove bad characters bad_chars = '!?()"—–<>⌈⌋⌊[]+ʾ/;' processed_text = processed_text.translate(str.maketrans('', '', bad_chars)) processed_text = processed_text.replace('\x00GAP\x00', ' ').replace('\x00BIG\x00', ' ') # Handle fractions frac_map = { r'\.5\b': ' ½', r'\.25\b': ' ¼', r'\.75\b': ' ¾', r'\.33+\d*\b': ' ⅓', r'\.66+\d*\b': ' ⅔' } for pat, rep in frac_map.items(): processed_text = re.sub(r'(\d+)' + pat, r'\1' + rep, processed_text) processed_text = re.sub(r'\b0' + pat, rep.strip(), processed_text) # Remove repeated words - KEY FIX processed_text = re.sub(r'\b(\w+)(?:\s+\1\b)+', r'\1', processed_text) # Remove repeated phrases WITH punctuation (e.g., "The city said: The city said:") # Match phrases up to 8 words that may include punctuation for n in range(8, 1, -1): # Pattern: words possibly followed by colon/comma, repeated pat = r'((?:[\w,]+\s*:?\s*){' + str(n) + r'})(?:\s*\1)+' processed_text = re.sub(pat, r'\1', processed_text, flags=re.IGNORECASE) # Also handle exact phrase repetition without word boundaries for n in range(4, 1, -1): pat = r'\b((?:\w+\s+){' + str(n-1) + r'}\w+)(?:\s+\1\b)+' processed_text = re.sub(pat, r'\1', processed_text) # Final cleanup: remove any "X X X X" patterns (same word 3+ times) processed_text = re.sub(r'\b(\w+)(\s+\1){2,}\b', r'\1', processed_text) return re.sub(r'\s+', ' ', processed_text).strip().strip('-') class AkkadianTranslationDataset(Dataset): def __init__(self, dataframe): self.data = dataframe.reset_index(drop=True) def __len__(self): return len(self.data) def __getitem__(self, idx): row = self.data.iloc[idx] return row['id'], row['transliteration'] def create_model_soup(): """Create model soup by weight averaging - EXACT copy from winning kernel.""" total_score = sum(CONFIG['model_weights']) WEIGHTS = [w / total_score for w in CONFIG['model_weights']] print(f"Creating model soup with weights: {WEIGHTS}") # Use the second model as the base template template_model = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][1]) soup_sd = template_model.state_dict() model_1_sd = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][0]).state_dict() model_3_sd = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][2]).state_dict() for key in soup_sd: # Initialize with weighted value from template model (model 2) weighted_value = WEIGHTS[1] * soup_sd[key] norm_factor = WEIGHTS[1] # Add contributions from other models if key exists if key in model_1_sd: weighted_value += WEIGHTS[0] * model_1_sd[key] norm_factor += WEIGHTS[0] if key in model_3_sd: weighted_value += WEIGHTS[2] * model_3_sd[key] norm_factor += WEIGHTS[2] soup_sd[key] = weighted_value / norm_factor template_model.load_state_dict(soup_sd) # Free memory del model_1_sd, model_3_sd torch.cuda.empty_cache() return template_model.to(CONFIG['device']).eval().float() def main(): print("=" * 60) print("Deep Past - ByT5 Model Soup (Offline)") print("=" * 60) print(f"Device: {CONFIG['device']}") # Verify model paths exist print("\nVerifying model paths...") for i, model_path in enumerate(CONFIG['models']): if os.path.exists(model_path): print(f" [{i+1}] {model_path}: OK") else: print(f" [{i+1}] {model_path}: MISSING!") input_dir = "/kaggle/input" if os.path.exists(input_dir): print(f" Available inputs: {os.listdir(input_dir)}") # Load test data print(f"\nLoading test data from {CONFIG['data_path']}...") dataframe = pd.read_csv(CONFIG['data_path']) print(f"Loaded {len(dataframe)} samples") # Preprocess dataframe['transliteration'] = dataframe['transliteration'].apply(preprocess_transliteration) # Create model soup print("\nCreating model soup...") model = create_model_soup() tokenizer = AutoTokenizer.from_pretrained(CONFIG['models'][1]) print("Model soup created!") # Create dataloader data_loader = DataLoader( AkkadianTranslationDataset(dataframe), batch_size=CONFIG['batch_size'], shuffle=False, num_workers=2, collate_fn=lambda batch: ( [item[0] for item in batch], tokenizer( [item[1] for item in batch], max_length=CONFIG['max_len'], padding=True, truncation=True, return_tensors="pt" ) ) ) # Inference print(f"\nRunning inference ({len(dataframe)} samples)...") inference_results = [] with torch.inference_mode(): for ids, inputs in tqdm(data_loader, desc="Translating"): outputs = model.generate( input_ids=inputs.input_ids.to(CONFIG['device']), attention_mask=inputs.attention_mask.to(CONFIG['device']), **CONFIG['gen_params'] ) decoded_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True) cleaned_translations = [postprocess_translation(text) for text in decoded_texts] inference_results.extend(zip(ids, cleaned_translations)) # Create submission submission = pd.DataFrame(inference_results, columns=['id', 'translation']) # Validate print(f"\nSubmission shape: {submission.shape}") print(f"Columns: {list(submission.columns)}") print(f"Non-empty translations: {sum(1 for t in submission['translation'] if t)}") # Save output_path = "/kaggle/working/submission.csv" submission.to_csv(output_path, index=False) print(f"Saved to {output_path}") # Show samples print("\n" + "=" * 60) print("Sample outputs:") for i in range(min(5, len(submission))): print(f"\nID: {submission.iloc[i]['id']}") print(f"Input: {dataframe.iloc[i]['transliteration'][:60]}...") print(f"Output: {submission.iloc[i]['translation'][:200]}") print("\n" + "=" * 60) print("Done!") if __name__ == "__main__": main()