{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"kernelVersion","sourceId":294739278}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"733c058f","cell_type":"markdown","source":"# Akkadian to English Machine Translation - Seq2Seq Model Training\n\nThis notebook fine-tunes a seq2seq model (ByT5) for Akkadian to English translation using the training dataset.\n\n**Key Features:**\n- Optimized text preprocessing with Akkadian-specific normalization\n- Seq2Seq model fine-tuning with HuggingFace Transformers\n- Flexible training with fallback PyTorch loop\n- Model checkpoint saving for submission","metadata":{}},{"id":"0e6657cc","cell_type":"code","source":"# Install required libraries\n!pip install transformers torch pandas -q","metadata":{},"outputs":[],"execution_count":null},{"id":"cee81d1d","cell_type":"code","source":"import os\nimport logging\nfrom pathlib import Path\nfrom typing import List\nimport random\n\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n)\nimport re\n\n# Setup logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"CUDA device: {torch.cuda.get_device_name(0)}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"c88bf1ad","cell_type":"code","source":"class OptimizedPreprocessor:\n    \"\"\"Standalone minimal variant of the notebook's preprocessor.\n    Implements the most important normalization and gap handling so training data\n    will match inference preprocessing.\n    \"\"\"\n\n    def __init__(self):\n        self.patterns = {\n            \"certain_reading\": re.compile(r\"!\"),\n            \"uncertain_reading\": re.compile(r\"\\?\"),\n            \"line_divider\": re.compile(r\"/\"),\n            \"word_divider\": re.compile(r\"[:.]\"),\n            \"scribal_insertions_brackets\": re.compile(r\"<([^>]*)>\"),\n            \"errant_signs_double_brackets\": re.compile(r\"<<([^>]*)>>\"),\n            \"partial_break_signs\": re.compile(r\"[˹˺]\"),\n            \"clear_break_brackets\": re.compile(r\"\\[([^\\]]*)\\]\"),\n            \"comments_parentheses\": re.compile(r\"\\([^)]*\\)\"),\n            \"small_gap_marker\": re.compile(r\"\\[x\\]\"),\n            \"ellipsis_gaps\": re.compile(r\"(\\.{3,}|…+|……|\\[…\\s*…\\])\"),\n            \"multiple_x_gaps\": re.compile(r\"(xx+|\\s+x\\s+)\") ,\n            \"h_normalization\": re.compile(r\"[ḫḪ]\"),\n            \"subscript_numbers\": re.compile(r\"(\\w)([₀-₉ₓ]+)\"),\n            \"line_numbers\": re.compile(r\"^\\s*\\d+['\\\"]*\\s*\"),\n            \"line_numbers_mid\": re.compile(r\"\\s+\\d+['\\\"]+\\s+\"),\n        }\n\n        self.unicode_chars = {\n            'á': 'a2', 'à': 'a3', 'é': 'e2', 'è': 'e3',\n            'í': 'i2', 'ì': 'i3', 'ú': 'u2', 'ù': 'u3',\n            'š': 'sz', 'Š': 'SZ', 'ṣ': 's,', 'Ṣ': 'S,',\n            'ṭ': 't,', 'Ṭ': 'T,', 'ḫ': 'h', 'Ḫ': 'H',\n            '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n            '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9', 'ₓ': 'x'\n        }\n\n    def preprocess_text(self, text: str) -> str:\n        if pd.isna(text) or not text:\n            return \"\"\n        text = str(text).strip()\n        text = self.patterns['certain_reading'].sub('', text)\n        text = self.patterns['uncertain_reading'].sub('', text)\n        text = self.patterns['line_divider'].sub(' ', text)\n        text = self.patterns['word_divider'].sub(' ', text)\n        text = self.patterns['scribal_insertions_brackets'].sub(r\"\\1\", text)\n        text = self.patterns['errant_signs_double_brackets'].sub('', text)\n        text = self.patterns['comments_parentheses'].sub('', text)\n        text = self.patterns['partial_break_signs'].sub('', text)\n        text = self.patterns['clear_break_brackets'].sub(r\"\\1\", text)\n        text = self.patterns['line_numbers'].sub('', text)\n        text = self.patterns['line_numbers_mid'].sub(' ', text)\n        text = self.patterns['small_gap_marker'].sub('<gap>', text)\n        text = self.patterns['ellipsis_gaps'].sub('<big_gap>', text)\n        text = self.patterns['multiple_x_gaps'].sub('<gap>', text)\n        text = self.patterns['h_normalization'].sub(lambda m: 'h' if m.group().islower() else 'H', text)\n\n        def normalize_subscripts(match):\n            base = match.group(1)\n            subscript = match.group(2)\n            normal_nums = subscript.translate(str.maketrans('₀₁₂₃₄₅₆₇₈₉ₓ', '0123456789x'))\n            return base + normal_nums\n\n        text = self.patterns['subscript_numbers'].sub(normalize_subscripts, text)\n        for k, v in self.unicode_chars.items():\n            text = text.replace(k, v)\n        text = ' '.join(text.split())\n        return text\n\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        return [self.preprocess_text(t) for t in texts]\n\nprint(\"✓ OptimizedPreprocessor class defined\")","metadata":{},"outputs":[],"execution_count":null},{"id":"a4f2e283","cell_type":"code","source":"class TranslationDataset(Dataset):\n    def __init__(self, inputs, targets, tokenizer, max_length=512):\n        self.inputs = inputs\n        self.targets = targets\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.inputs)\n\n    def __getitem__(self, idx):\n        inp = self.inputs[idx]\n        tgt = self.targets[idx]\n        model_inputs = self.tokenizer(\n            inp,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        # ByT5Tokenizer doesn't have as_target_tokenizer(), so tokenize directly\n        labels = self.tokenizer(\n            tgt,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n\n        input_ids = model_inputs.input_ids.squeeze()\n        attention_mask = model_inputs.attention_mask.squeeze()\n        labels_ids = labels.input_ids.squeeze()\n        # Replace pad token id's in labels by -100 so they are ignored by loss\n        labels_ids[labels_ids == self.tokenizer.pad_token_id] = -100\n\n        return {\n            'input_ids': input_ids,\n            'attention_mask': attention_mask,\n            'labels': labels_ids\n        }\n\nprint(\"✓ TranslationDataset class defined\")","metadata":{},"outputs":[],"execution_count":null},{"id":"22ed2257","cell_type":"code","source":"def load_data(path: str):\n    \"\"\"Load and validate the CSV training data.\"\"\"\n    df = pd.read_csv(path)\n\n    # Accept common alternative id column names (e.g. 'oare_id') by normalizing to 'id'\n    if 'id' not in df.columns and 'oare_id' in df.columns:\n        df = df.rename(columns={'oare_id': 'id'})\n        logger.info(\"Renamed column 'oare_id' -> 'id' for compatibility\")\n\n    # Check for minimum required columns\n    required_cols = {'transliteration', 'translation'}\n    available_cols = set(df.columns)\n    if not required_cols.issubset(available_cols):\n        available = ','.join(df.columns.tolist())\n        raise ValueError(f\"CSV must contain columns: {required_cols} (available: {available})\")\n    \n    df = df.dropna(subset=['transliteration', 'translation'])\n    return df\n\ndef validate_kaggle_input_dir(input_dir: str) -> dict:\n    \"\"\"Validate Kaggle input directory structure and list available datasets.\"\"\"\n    available_files = {\n        'train.csv': False,\n        'test.csv': False,\n        'sentence_alignment.csv': False,\n        'Sentences_Oare_FirstWord_LinNum.csv': False,\n        'published_texts.csv': False,\n        'eBL_Dictionary.csv': False,\n        'OA_Lexicon_eBL.csv': False,\n    }\n    \n    if not os.path.exists(input_dir):\n        logger.warning(f\"Input directory not found: {input_dir}\")\n        return available_files\n    \n    for filename in available_files.keys():\n        filepath = os.path.join(input_dir, filename)\n        available_files[filename] = os.path.exists(filepath)\n    \n    return available_files\n\ndef simple_train_test_split(inputs, targets, test_size=0.05, random_state=None):\n    \"\"\"Simple reproducible train/validation split to avoid scikit-learn dependency.\"\"\"\n    if random_state is not None:\n        random.seed(int(random_state))\n\n    combined = list(zip(inputs, targets))\n    n = len(combined)\n    if n == 0:\n        return [], [], [], []\n\n    random.shuffle(combined)\n    test_n = int(max(1, round(n * float(test_size)))) if n > 1 else 0\n\n    # Ensure at least one training sample if possible\n    if test_n >= n:\n        test_n = max(1, n // 10)\n\n    test = combined[:test_n]\n    train = combined[test_n:]\n\n    if train:\n        train_inputs, train_targets = zip(*train)\n        train_inputs, train_targets = list(train_inputs), list(train_targets)\n    else:\n        train_inputs, train_targets = [], []\n\n    if test:\n        val_inputs, val_targets = zip(*test)\n        val_inputs, val_targets = list(val_inputs), list(val_targets)\n    else:\n        val_inputs, val_targets = [], []\n\n    return train_inputs, val_inputs, train_targets, val_targets\n\nprint(\"✓ Helper functions defined\")","metadata":{},"outputs":[],"execution_count":null},{"id":"38de0581","cell_type":"code","source":"# Configuration Parameters\nimport os\nfrom pathlib import Path\n\n# Detect Kaggle environment\nIS_KAGGLE = os.path.exists('/kaggle/input')\nprint(f\"Running in Kaggle environment: {IS_KAGGLE}\")\nprint(f\"Current working directory: {os.getcwd()}\\n\")\n\n# Kaggle input directory\nINPUT_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\ndata_file = None\n\nif IS_KAGGLE:\n    # In Kaggle environment, use /kaggle/input directly\n    print(\"Checking Kaggle input directory structure...\")\n    available_files = validate_kaggle_input_dir(INPUT_DIR)\n    print(f\"\\nAvailable files in {INPUT_DIR}:\")\n    for filename, exists in available_files.items():\n        status = \"✓\" if exists else \"✗\"\n        print(f\"  {status} {filename}\")\n    \n    # Try to find train.csv in Kaggle inputs\n    data_file = f\"{INPUT_DIR}/train.csv\"\n    if os.path.exists(data_file):\n        print(f\"\\n✓ Found Kaggle input directory at: {INPUT_DIR}\")\n        print(f\"✓ Using train file: {data_file}\")\n    else:\n        print(f\"\\n✗ train.csv not found at {INPUT_DIR}\")\n        data_file = None\nelse:\n    # Local development environment\n    print(\"Local development environment detected\\n\")\n    \n    # Try multiple common paths for local development\n    possible_paths = [\n        '../input/deep-past-initiative-machine-translation/train.csv',  # Kaggle notebooks\n        '../input/akkadian-english-machine-translation/train.csv',      # Alt Kaggle\n        './data/train.csv',                                              # Current directory\n        '../data/train.csv',                                             # One level up\n        '../../data/train.csv',                                          # Two levels up\n    ]\n\n    for path in possible_paths:\n        if os.path.exists(path):\n            data_file = os.path.abspath(path)\n            print(f\"✓ Found data file at: {path}\")\n            break\n\n    # If still not found, search in parent directories\n    if data_file is None:\n        current = Path.cwd()\n        for parent in [current] + list(current.parents):\n            train_csv = parent / 'data' / 'train.csv'\n            if train_csv.exists():\n                data_file = str(train_csv)\n                print(f\"✓ Found data file at: {data_file}\")\n                break\n\n    if data_file is None:\n        data_file = './data/train.csv'\n        print(\"Warning: Using default path. Ensure train.csv is in data/ folder.\")\n\n# Determine best local model path\ndef find_local_model(fallback_paths=None):\n    \"\"\"Find locally available pre-trained model for offline loading.\"\"\"\n    if fallback_paths is None:\n        fallback_paths = [\n            '/kaggle/input/datasets/akkadian-byt5-chunky-v1-0-0-pytorch-default-v1' if IS_KAGGLE else None,\n            '/kaggle/input/datasets/akkadian-byt5-chunky-v1' if IS_KAGGLE else None,\n            '/kaggle/input/datasets/byt5-base-transformers-default-v1' if IS_KAGGLE else None,\n            '/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x' if IS_KAGGLE else None,\n            '../models/akkadian-byt5-chunky-v1-0-0-pytorch-default-v1',\n            '../models/byt5-base-transformers-default-v1',\n            '../../models/akkadian-byt5-chunky-v1-0-0-pytorch-default-v1',\n            '../../models/byt5-base-transformers-default-v1',\n            './models/akkadian-byt5-chunky-v1-0-0-pytorch-default-v1',\n            './models/byt5-base-transformers-default-v1',\n            'models/akkadian-byt5-chunky-v1-0-0-pytorch-default-v1',\n            'models/byt5-base-transformers-default-v1',\n        ]\n    \n    fallback_paths = [p for p in fallback_paths if p is not None]\n    \n    for path in fallback_paths:\n        if os.path.exists(path) and os.path.exists(os.path.join(path, 'config.json')):\n            print(f\"✓ Found local model at: {path}\")\n            return path\n    \n    print(\"⚠️  No local model found. Will attempt HuggingFace Hub download.\")\n    return 'google/byt5-base'\n\nmodel_path = find_local_model()\n\nCONFIG = {\n    'is_kaggle': IS_KAGGLE,\n    'input_dir': INPUT_DIR,\n    'train_file': data_file,\n    'model_name': model_path,\n    'model_is_local': model_path != 'google/byt5-base',\n    'output_dir': './fine_tuned_model',\n    'epochs': 3,\n    'batch_size': 8,\n    'lr': 5e-5,\n    'max_length': 256,\n    'seed': 42,\n}\n\n# Set random seed for reproducibility\ntorch.manual_seed(CONFIG['seed'])\nrandom.seed(CONFIG['seed'])\n\nprint(f\"\\nConfiguration:\")\nprint(f\"  Environment: {'🟦 Kaggle' if CONFIG['is_kaggle'] else '💻 Local'}\")\nprint(f\"  Input directory: {CONFIG['input_dir']}\")\nprint(f\"  Data file: {CONFIG['train_file']}\")\nprint(f\"  Data file exists: {os.path.exists(CONFIG['train_file']) if CONFIG['train_file'] else False}\")\nprint(f\"  Current working directory: {os.getcwd()}\")\nprint(f\"  Model source: {'📁 Local (offline)' if CONFIG['model_is_local'] else '🌐 HuggingFace Hub'}\")\nfor key, value in CONFIG.items():\n    if key not in ['train_file', 'input_dir', 'is_kaggle', 'model_is_local']:\n        print(f\"  {key}: {value}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"9820f19c","cell_type":"code","source":"# Load training data from Kaggle input or local filesystem\nprint(f\"Loading data from {CONFIG['train_file']}...\")\n\n# Fallback if train_file is None - search all available Kaggle datasets\nif CONFIG['train_file'] is None:\n    print(\"\\n⚠️  train_file is None. Searching for datasets in /kaggle/input...\")\n    \n    # Search for train.csv in any subdirectory under /kaggle/input\n    if os.path.exists('/kaggle/input'):\n        for root, dirs, files in os.walk('/kaggle/input'):\n            if 'train.csv' in files:\n                CONFIG['train_file'] = os.path.join(root, 'train.csv')\n                print(f\"✓ Found train.csv at: {CONFIG['train_file']}\")\n                break\n    \n    # If still not found, use local fallback\n    if CONFIG['train_file'] is None:\n        CONFIG['train_file'] = './data/train.csv'\n        print(f\"⚠️  Using local fallback: {CONFIG['train_file']}\")\n\nif not os.path.exists(CONFIG['train_file']):\n    print(f\"\\n✗ ERROR: Data file not found at {CONFIG['train_file']}\")\n    print(f\"\\nDEBUG INFO:\")\n    print(f\"  Current working directory: {os.getcwd()}\")\n    print(f\"  Absolute path: {os.path.abspath(CONFIG['train_file'])}\")\n    print(f\"\\nAvailable files in current directory:\")\n    for f in sorted(os.listdir('.'))[:10]:\n        print(f\"  - {f}\")\n    if os.path.exists('..'):\n        print(f\"\\nAvailable files in parent directory:\")\n        for f in sorted(os.listdir('..'))[:10]:\n            print(f\"  - {f}\")\n    \n    # List kaggle/input structure if available\n    if os.path.exists('/kaggle/input'):\n        print(f\"\\nAvailable in /kaggle/input:\")\n        for item in sorted(os.listdir('/kaggle/input'))[:10]:\n            print(f\"  - {item}\")\n    \n    raise FileNotFoundError(f\"Train file not found: {CONFIG['train_file']}\")\n\ndf = load_data(CONFIG['train_file'])\nprint(f\"✓ Loaded {len(df)} primary samples\")\n\n# Try to load additional data sources\nadditional_dfs = []\n\n# Determine data directory for additional files\ndata_dir = os.path.dirname(os.path.abspath(CONFIG['train_file']))\nprint(f\"\\nLooking for additional data sources in: {data_dir}\")\n\n# Check for sentence alignment data\nsent_align_candidates = [\n    f\"{data_dir}/sentence_alignment.csv\",\n    f\"{data_dir}/Sentences_Oare_FirstWord_LinNum.csv\",  # Alternative name\n]\n\nfor sent_align_path in sent_align_candidates:\n    if os.path.exists(sent_align_path):\n        try:\n            sent_df = pd.read_csv(sent_align_path)\n            # Handle different column names\n            if \"translit_sent\" in sent_df.columns and \"translation_sent_train\" in sent_df.columns:\n                sent_df_clean = sent_df.rename(columns={\n                    \"translit_sent\": \"transliteration\",\n                    \"translation_sent_train\": \"translation\"\n                }, inplace=False)\n            elif \"transliteration\" in sent_df.columns and \"translation\" in sent_df.columns:\n                sent_df_clean = sent_df\n            else:\n                continue\n            \n            sent_df_clean = sent_df_clean[['transliteration', 'translation']]\n            sent_df_clean = sent_df_clean.dropna(subset=['transliteration', 'translation'])\n            if len(sent_df_clean) > 0:\n                additional_dfs.append(sent_df_clean)\n                print(f\"✓ Loaded {len(sent_df_clean)} samples from sentence alignment: {os.path.basename(sent_align_path)}\")\n                break\n        except Exception as e:\n            logger.warning(f\"Could not load sentence alignment from {sent_align_path}: {e}\")\n\n# Check for published texts\npub_text_path = f\"{data_dir}/published_texts.csv\"\nif os.path.exists(pub_text_path):\n    try:\n        pub_df = pd.read_csv(pub_text_path)\n        # Check if it has transliteration and translation columns\n        if 'transliteration' in pub_df.columns and 'translation' in pub_df.columns:\n            pub_df_clean = pub_df[['transliteration', 'translation']].copy()\n            pub_df_clean = pub_df_clean.dropna(subset=['transliteration', 'translation'])\n            if len(pub_df_clean) > 0:\n                additional_dfs.append(pub_df_clean)\n                print(f\"✓ Loaded {len(pub_df_clean)} samples from published texts\")\n    except Exception as e:\n        logger.warning(f\"Could not load published texts: {e}\")\n\n# Combine all dataframes\nif additional_dfs:\n    df = pd.concat([df] + additional_dfs, ignore_index=True)\n    print(f\"\\n✓ Combined dataset size: {len(df)} samples\")\n\n# Remove duplicates while preserving order\ndf_before = len(df)\ndf = df.drop_duplicates(subset=['transliteration', 'translation'], keep='first')\nif len(df) < df_before:\n    print(f\"✓ Removed {df_before - len(df)} duplicate entries\")\n\nprint(f\"\\nFinal Dataset Info:\")\nprint(f\"  Total samples: {len(df)}\")\nprint(f\"  Columns: {df.columns.tolist()}\")\nprint(f\"  Shape: {df.shape}\")\nprint(f\"  Missing values:\")\nprint(f\"    transliteration: {df['transliteration'].isna().sum()}\")\nprint(f\"    translation: {df['translation'].isna().sum()}\")\nprint(f\"\\nFirst 3 samples:\")\nprint(df[['transliteration', 'translation']].head(3))","metadata":{},"outputs":[],"execution_count":null},{"id":"b72da69a","cell_type":"code","source":"# Initialize preprocessor\nprint(\"Preprocessing transliterations...\")\npre = OptimizedPreprocessor()\n\n# Preprocess inputs\ninputs = pre.preprocess_batch(df['transliteration'].astype(str).tolist())\ninputs = [f\"translate Akkadian to English: {t}\" for t in inputs]\ntargets = df['translation'].astype(str).tolist()\n\nprint(f\"✓ Preprocessed {len(inputs)} inputs\")\nprint(f\"\\nExample (after preprocessing):\")\nprint(f\"  Input: {inputs[0][:80]}...\")\nprint(f\"  Target: {targets[0]}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"323ad3b7","cell_type":"code","source":"# Data quality analysis and statistics\nprint(\"\\n\" + \"=\"*60)\nprint(\"DATA QUALITY ANALYSIS\")\nprint(\"=\"*60)\n\n# Calculate length statistics\ninput_lengths = [len(inp.split()) for inp in inputs]\ntarget_lengths = [len(tgt.split()) for tgt in targets]\n\nprint(f\"\\nInput (transliteration + prefix) statistics:\")\nprint(f\"  Min length: {min(input_lengths)} tokens\")\nprint(f\"  Max length: {max(input_lengths)} tokens\")\nprint(f\"  Mean length: {sum(input_lengths)/len(input_lengths):.1f} tokens\")\nprint(f\"  Median length: {sorted(input_lengths)[len(input_lengths)//2]} tokens\")\n\nprint(f\"\\nTarget (translation) statistics:\")\nprint(f\"  Min length: {min(target_lengths)} tokens\")\nprint(f\"  Max length: {max(target_lengths)} tokens\")\nprint(f\"  Mean length: {sum(target_lengths)/len(target_lengths):.1f} tokens\")\nprint(f\"  Median length: {sorted(target_lengths)[len(target_lengths)//2]} tokens\")\n\n# Check for special characters and patterns\nproper_noun_count = sum(1 for t in df['transliteration'].astype(str).tolist() \n                        if any(c.isupper() for c in t.split()))\nsumerian_logogram_count = sum(1 for t in df['transliteration'].astype(str).tolist() \n                              if any(w.isupper() and len(w) > 1 for w in t.split()))\ngap_count = sum(1 for inp in inputs if '<gap>' in inp)\n\nprint(f\"\\nSpecial patterns:\")\nprint(f\"  Entries with proper nouns (capitalized): {proper_noun_count}\")\nprint(f\"  Entries with Sumerian logograms (ALL CAPS): {sumerian_logogram_count}\")\nprint(f\"  Entries with gaps (<gap>): {gap_count}\")\n\n# Show length distribution\nimport numpy as np\npercentiles = [25, 50, 75, 90, 95]\nprint(f\"\\nInput length percentiles:\")\nfor p in percentiles:\n    val = np.percentile(input_lengths, p)\n    print(f\"  {p}th: {val:.0f} tokens\")\n\nprint(f\"\\nTarget length percentiles:\")\nfor p in percentiles:\n    val = np.percentile(target_lengths, p)\n    print(f\"  {p}th: {val:.0f} tokens\")\n\nprint(\"\\n\" + \"=\"*60)","metadata":{},"outputs":[],"execution_count":null},{"id":"293a8a42","cell_type":"code","source":"# Train/validation split\ntrain_inputs, val_inputs, train_targets, val_targets = simple_train_test_split(\n    inputs, targets, test_size=0.05, random_state=CONFIG['seed']\n)\n\nprint(f\"Data split:\")\nprint(f\"  Training samples: {len(train_inputs)}\")\nprint(f\"  Validation samples: {len(val_inputs)}\")\nprint(f\"  Test size ratio: {len(val_inputs) / (len(train_inputs) + len(val_inputs)):.2%}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"e92736cc","cell_type":"code","source":"# Load tokenizer and model\nprint(f\"Loading tokenizer & model: {CONFIG['model_name']}...\")\nprint(f\"Model is local: {CONFIG['model_is_local']}\\n\")\n\ntry:\n    # Use local_files_only for offline loading\n    tokenizer = AutoTokenizer.from_pretrained(\n        CONFIG['model_name'],\n        local_files_only=CONFIG['model_is_local'],\n        trust_remote_code=True\n    )\n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        CONFIG['model_name'],\n        local_files_only=CONFIG['model_is_local'],\n        torch_dtype=torch.float32,\n        device_map='auto' if torch.cuda.is_available() else None,\n    )\n    print(f\"✓ Model loaded successfully\")\nexcept Exception as e:\n    print(f\"Error loading model: {e}\")\n    print(f\"Attempting to download from HuggingFace Hub...\")\n    tokenizer = AutoTokenizer.from_pretrained('google/byt5-base')\n    model = AutoModelForSeq2SeqLM.from_pretrained('google/byt5-base')\n    print(f\"✓ Model loaded from HuggingFace Hub\")\n\nprint(f\"\\nModel Info:\")\nprint(f\"  Model name: {CONFIG['model_name']}\")\nprint(f\"  Tokenizer vocab size: {tokenizer.vocab_size}\")\nprint(f\"  Model parameters: {sum(p.numel() for p in model.parameters()):,}\")\nif torch.cuda.is_available():\n    print(f\"  Model device: {next(model.parameters()).device}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"ed31966c","cell_type":"code","source":"# Create datasets\nprint(\"Creating datasets...\")\ntrain_dataset = TranslationDataset(\n    train_inputs, train_targets, tokenizer, max_length=CONFIG['max_length']\n)\nval_dataset = TranslationDataset(\n    val_inputs, val_targets, tokenizer, max_length=CONFIG['max_length']\n)\n\nprint(f\"✓ Datasets created\")\nprint(f\"  Train dataset size: {len(train_dataset)}\")\nprint(f\"  Validation dataset size: {len(val_dataset)}\")\n\n# Create data collator\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer, model=model, label_pad_token_id=-100, pad_to_multiple_of=None\n)\n\nprint(f\"✓ Data collator created\")","metadata":{},"outputs":[],"execution_count":null},{"id":"6906777c","cell_type":"code","source":"# Try to use Hugging Face Seq2SeqTrainer\ntrainer_success = False\ntry:\n    from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer\n    \n    logger.info(\"Seq2SeqTrainer available - using Hugging Face Trainer\")\n    print(\"Using Hugging Face Seq2SeqTrainer...\\n\")\n    \n    training_args = Seq2SeqTrainingArguments(\n        output_dir=CONFIG['output_dir'],\n        per_device_train_batch_size=2,  # Reduced from 8 for GPU memory\n        per_device_eval_batch_size=2,   # Reduced from 8\n        gradient_accumulation_steps=4,  # Accumulate gradients to simulate batch_size=8\n        predict_with_generate=False,    # Disable for training to save memory\n        eval_strategy='no',             # Disable eval during training to save memory\n        save_strategy='no',             # Disable checkpoint saving during training\n        logging_steps=100,\n        num_train_epochs=CONFIG['epochs'],\n        learning_rate=CONFIG['lr'],\n        bf16=torch.cuda.is_available() and torch.cuda.get_device_capability(0)[0] >= 8,  # Use bf16 if available\n        fp16=torch.cuda.is_available() and torch.cuda.get_device_capability(0)[0] < 8,   # Use fp16 otherwise\n        optim='adafactor',              # Memory-efficient optimizer\n        max_grad_norm=1.0,\n        weight_decay=0.01,\n        warmup_ratio=0.1,\n        save_total_limit=1,\n        remove_unused_columns=False,\n        push_to_hub=False,\n        dataloader_num_workers=0,\n        dataloader_pin_memory=True,\n    )\n    \n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset,\n        data_collator=data_collator,\n        # Note: tokenizer parameter removed as it's not supported in this version\n    )\n    \n    print(\"Starting training with Seq2SeqTrainer...\")\n    trainer.train()\n    trainer_success = True\n    print(\"\\n✓ Training completed with Seq2SeqTrainer\")\n    \nexcept Exception as e:\n    logger.warning(f\"Seq2SeqTrainer not available or failed: {e}\")\n    print(f\"\\nSeq2SeqTrainer initialization failed: {e}\")\n    print(\"Proceeding with fallback training...\")","metadata":{},"outputs":[],"execution_count":null},{"id":"0dc954fc","cell_type":"code","source":"def run_fallback_training(model, tokenizer, train_dataset, val_dataset, config, device=None):\n    \"\"\"Simple PyTorch training loop with memory optimization for limited VRAM.\"\"\"\n    if device is None:\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    print(f\"\\nUsing fallback training on {device}\")\n    model.to(device)\n    \n    # Use much smaller batch size for GPU memory constraints\n    batch_size = 2  # Reduced from 8\n    gradient_accumulation_steps = 4  # Accumulate gradients to simulate larger batch\n    \n    train_loader = DataLoader(\n        train_dataset, batch_size=batch_size, shuffle=True, num_workers=0\n    )\n    val_loader = (\n        DataLoader(\n            val_dataset, batch_size=batch_size, shuffle=False, num_workers=0\n        ) if val_dataset else None\n    )\n    \n    # Use memory-efficient optimizer\n    try:\n        from transformers import Adafactor\n        optimizer = Adafactor(\n            model.parameters(),\n            scale_parameter=False,\n            relative_step=True,\n            warmup_init=True,\n        )\n    except:\n        optimizer = torch.optim.AdamW(model.parameters(), lr=config['lr'])\n    \n    # Enable gradient checkpointing to save memory\n    if hasattr(model, 'gradient_checkpointing_enable'):\n        model.gradient_checkpointing_enable()\n    \n    for epoch in range(int(config['epochs'])):\n        model.train()\n        running_loss = 0.0\n        accumulated_loss = 0.0\n        print(f\"\\nEpoch {epoch+1}/{config['epochs']}\")\n        \n        for step, batch in enumerate(train_loader):\n            try:\n                input_ids = batch['input_ids'].to(device)\n                attention_mask = batch['attention_mask'].to(device)\n                labels = batch['labels'].to(device)\n                \n                # Forward pass\n                with torch.cuda.amp.autocast() if device.type == 'cuda' else torch.no_grad():\n                    outputs = model(\n                        input_ids=input_ids, \n                        attention_mask=attention_mask, \n                        labels=labels\n                    )\n                    loss = outputs.loss\n                \n                # Backward pass with gradient accumulation\n                if loss is not None:\n                    loss = loss / gradient_accumulation_steps\n                    loss.backward()\n                    accumulated_loss += loss.item()\n                \n                # Optimizer step and gradient zeroing\n                if (step + 1) % gradient_accumulation_steps == 0:\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                    optimizer.step()\n                    optimizer.zero_grad()\n                    \n                    running_loss += accumulated_loss\n                    accumulated_loss = 0.0\n                    \n                    if ((step + 1) // gradient_accumulation_steps) % 50 == 0:\n                        avg = running_loss / ((step + 1) // gradient_accumulation_steps)\n                        print(f\"  Step {(step + 1) // gradient_accumulation_steps} - avg loss: {avg:.4f}\")\n                \n                # Clear CUDA cache periodically\n                if (step + 1) % 100 == 0 and device.type == 'cuda':\n                    torch.cuda.empty_cache()\n                    \n            except RuntimeError as e:\n                if 'out of memory' in str(e).lower():\n                    print(f\"\\n⚠️  OOM at step {step}, clearing cache and continuing...\")\n                    torch.cuda.empty_cache()\n                    optimizer.zero_grad()\n                    continue\n                else:\n                    raise\n        \n        # Validation\n        if val_loader is not None:\n            model.eval()\n            try:\n                val_loss_sum = 0.0\n                val_steps = 0\n                with torch.no_grad():\n                    for val_batch in val_loader:\n                        try:\n                            v_in = val_batch['input_ids'].to(device)\n                            v_att = val_batch['attention_mask'].to(device)\n                            v_lbl = val_batch['labels'].to(device)\n                            v_out = model(input_ids=v_in, attention_mask=v_att, labels=v_lbl)\n                            if v_out.loss is not None:\n                                val_loss_sum += v_out.loss.item()\n                                val_steps += 1\n                        except RuntimeError as e:\n                            if 'out of memory' in str(e).lower():\n                                torch.cuda.empty_cache()\n                                continue\n                            raise\n                if val_steps > 0:\n                    print(f\"  Validation avg loss: {val_loss_sum / val_steps:.4f}\")\n            except Exception as e:\n                print(f\"  Validation step skipped: {e}\")\n    \n    os.makedirs(config['output_dir'], exist_ok=True)\n    print(f\"\\nSaving fallback-trained model to {config['output_dir']}\")\n    model.save_pretrained(config['output_dir'])\n    tokenizer.save_pretrained(config['output_dir'])\n    print(\"✓ Model and tokenizer saved\")\n\nif not trainer_success:\n    try:\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        # Clear CUDA cache before training\n        if device.type == 'cuda':\n            torch.cuda.empty_cache()\n            torch.cuda.reset_peak_memory_stats()\n        run_fallback_training(model, tokenizer, train_dataset, val_dataset, CONFIG, device=device)\n    except Exception as e:\n        print(f\"Fallback training also failed: {e}\")\n        # Try to at least save the untrained model\n        try:\n            os.makedirs(CONFIG['output_dir'], exist_ok=True)\n            model.save_pretrained(CONFIG['output_dir'])\n            tokenizer.save_pretrained(CONFIG['output_dir'])\n            print(\"⚠️  Saved untrained model due to training failure\")\n        except:\n            pass\n        raise","metadata":{},"outputs":[],"execution_count":null},{"id":"97d5136f","cell_type":"code","source":"# Ensure model and tokenizer are saved\nprint(f\"\\nEnsuring model/tokenizer saved to {CONFIG['output_dir']}\")\nif not os.path.exists(CONFIG['output_dir']):\n    os.makedirs(CONFIG['output_dir'], exist_ok=True)\n\ntry:\n    model.save_pretrained(CONFIG['output_dir'])\n    tokenizer.save_pretrained(CONFIG['output_dir'])\n    print(\"✓ Model and tokenizer saved successfully\")\n    \n    # List saved files\n    saved_files = os.listdir(CONFIG['output_dir'])\n    print(f\"\\nSaved files:\")\n    for f in saved_files:\n        print(f\"  - {f}\")\nexcept Exception as e:\n    print(f\"Warning: Could not save model: {e}\")\n\nprint(\"\\n✓ Training and model saving completed!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"f1c90d36","cell_type":"code","source":"# Verification: Load and test the saved model\nprint(\"\\n\" + \"=\"*50)\nprint(\"Model Verification & Kaggle Submission\")\nprint(\"=\"*50)\n\ntry:\n    # Load saved model\n    print(\"\\nLoading saved model for verification...\")\n    saved_tokenizer = AutoTokenizer.from_pretrained(CONFIG['output_dir'])\n    saved_model = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['output_dir'])\n    \n    print(f\"✓ Successfully loaded model from {CONFIG['output_dir']}\")\n    \n    # Test inference\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    saved_model.to(device)\n    saved_model.eval()\n    \n    # Example inference\n    test_input = \"translate Akkadian to English: nam-inim-ba\"\n    input_ids = saved_tokenizer(test_input, return_tensors='pt').input_ids.to(device)\n    \n    with torch.no_grad():\n        outputs = saved_model.generate(input_ids, max_length=50)\n    \n    predicted_text = saved_tokenizer.decode(outputs[0], skip_special_tokens=True)\n    \n    print(f\"\\nExample Inference Test:\")\n    print(f\"  Input:  {test_input}\")\n    print(f\"  Output: {predicted_text}\")\n    print(f\"\\n✓ Model inference working correctly!\")\n    \nexcept Exception as e:\n    print(f\"Verification failed: {e}\")\n\nprint(f\"\\n\" + \"=\"*50)\nprint(\"KAGGLE SUBMISSION READY\")\nprint(\"=\"*50)\nprint(f\"\\n✓ Notebook completed successfully!\")\nprint(f\"\\n📁 Model Location: {CONFIG['output_dir']}\")\nprint(f\"\\n📝 To submit to Kaggle:\")\nprint(f\"   1. Copy the '{CONFIG['output_dir']}' folder\")\nprint(f\"   2. Upload as a dataset or use it in your submission notebook\")\nprint(f\"   3. Load model using:\")\nprint(f\"      from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\")\nprint(f\"      tokenizer = AutoTokenizer.from_pretrained('./fine_tuned_model')\")\nprint(f\"      model = AutoModelForSeq2SeqLM.from_pretrained('./fine_tuned_model')\")\nprint(f\"\\n\" + \"=\"*50)","metadata":{},"outputs":[],"execution_count":null}]}