{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":710695,"sourceType":"modelInstanceVersion","modelInstanceId":539934,"modelId":553129}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\" # Use only the first GPU\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:28:50.340132Z","iopub.execute_input":"2026-01-15T14:28:50.340438Z","iopub.status.idle":"2026-01-15T14:28:50.344157Z","shell.execute_reply.started":"2026-01-15T14:28:50.340394Z","shell.execute_reply":"2026-01-15T14:28:50.343612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install sacrebleu -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:28:50.345454Z","iopub.execute_input":"2026-01-15T14:28:50.345973Z","iopub.status.idle":"2026-01-15T14:28:54.886253Z","shell.execute_reply.started":"2026-01-15T14:28:50.345933Z","shell.execute_reply":"2026-01-15T14:28:54.885361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"WANDB_DISABLED\"] = \"true\"\nos.environ[\"WANDB_MODE\"] = \"disabled\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:28:54.887760Z","iopub.execute_input":"2026-01-15T14:28:54.887974Z","iopub.status.idle":"2026-01-15T14:28:54.891798Z","shell.execute_reply.started":"2026-01-15T14:28:54.887952Z","shell.execute_reply":"2026-01-15T14:28:54.890986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, AutoModelForSeq2SeqLM, \n    Seq2SeqTrainingArguments, Seq2SeqTrainer,\n    DataCollatorForSeq2Seq, EarlyStoppingCallback\n)\nfrom sklearn.model_selection import train_test_split\nfrom nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\nimport gc\nimport optuna\nfrom optuna.pruners import HyperbandPruner\n\nprint(\"PyTorch:\", torch.__version__)\nprint(f\"Device: {torch.device('cuda' if torch.cuda.is_available() else 'cpu')}\")\nprint(f\"GPUs available: {torch.cuda.device_count()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:28:54.892509Z","iopub.execute_input":"2026-01-15T14:28:54.892783Z","iopub.status.idle":"2026-01-15T14:29:29.033483Z","shell.execute_reply.started":"2026-01-15T14:28:54.892760Z","shell.execute_reply":"2026-01-15T14:29:29.032855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n\n    text = re.sub(r'xx', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'……', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n\n    return text\n\n\ndef replace_gaps_back(text):\n    if pd.isna(text):  \n        return text\n    \n    text = re.sub(r'<gap>', 'x', text)\n    text = re.sub(r'<big_gap>', '...', text)\n\n    return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.035036Z","iopub.execute_input":"2026-01-15T14:29:29.035631Z","iopub.status.idle":"2026-01-15T14:29:29.041235Z","shell.execute_reply.started":"2026-01-15T14:29:29.035603Z","shell.execute_reply":"2026-01-15T14:29:29.040530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_PATH = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\ntrain_df = pd.read_csv(DATA_PATH / \"train.csv\")\ntest_df = pd.read_csv(DATA_PATH / \"test.csv\")\n\nprint(f\"Original train shape: {train_df.shape}\")\n\n# Apply gap preprocessing\ntrain_df['src'] = \"translate Akkadian to English: \" + train_df['transliteration'].astype(str).apply(replace_gaps)\ntrain_df['tgt'] = train_df['translation'].astype(str).str.strip().str.lower()\n\n# Filter by length\ntrain_df = train_df[(train_df.src.str.len() >= 10) & (train_df.src.str.len() <= 1000)]\ntrain_df = train_df[(train_df.tgt.str.len() >= 5) & (train_df.tgt.str.len() <= 500)]\n\n# Remove duplicates\ntrain_df = train_df.drop_duplicates(subset=['src', 'tgt']).reset_index(drop=True)\n\nprint(f\"Cleaned train shape: {train_df.shape}\")\ntrain_df.head(3)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.042365Z","iopub.execute_input":"2026-01-15T14:29:29.042699Z","iopub.status.idle":"2026-01-15T14:29:29.186609Z","shell.execute_reply.started":"2026-01-15T14:29:29.042667Z","shell.execute_reply":"2026-01-15T14:29:29.185921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_texts, val_texts = train_test_split(\n    train_df[['src', 'tgt']], \n    test_size=0.1, \n    random_state=42\n)\n\nprint(f\"Train samples: {len(train_texts)}, Val samples: {len(val_texts)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.187459Z","iopub.execute_input":"2026-01-15T14:29:29.187741Z","iopub.status.idle":"2026-01-15T14:29:29.196436Z","shell.execute_reply.started":"2026-01-15T14:29:29.187719Z","shell.execute_reply":"2026-01-15T14:29:29.195585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/input/byt5-small/pytorch/default/1/byt5-small\"  # Upload google/byt5-small as dataset\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, use_fast=False)\nMAX_LEN = 512\n\ndef preprocess_function(examples):\n    inputs = tokenizer(\n        examples[\"src\"], \n        max_length=MAX_LEN, \n        truncation=True, \n        padding=False\n    )\n    labels = tokenizer(\n        examples[\"tgt\"], \n        max_length=MAX_LEN, \n        truncation=True, \n        padding=False\n    )\n    inputs[\"labels\"] = labels[\"input_ids\"]\n    return inputs\n\n# Convert to HF Dataset and tokenize\ntrain_ds = Dataset.from_pandas(train_texts)\nval_ds = Dataset.from_pandas(val_texts)\n\ntrain_ds = train_ds.map(preprocess_function, batched=True, remove_columns=['src', 'tgt'])\nval_ds = val_ds.map(preprocess_function, batched=True, remove_columns=['src', 'tgt'])\n\nprint(f\"Tokenized - Train: {len(train_ds)}, Val: {len(val_ds)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.197291Z","iopub.execute_input":"2026-01-15T14:29:29.197532Z","iopub.status.idle":"2026-01-15T14:29:29.909732Z","shell.execute_reply.started":"2026-01-15T14:29:29.197511Z","shell.execute_reply":"2026-01-15T14:29:29.908977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sacrebleu\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    \n    # 1. Decode predictions\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    \n    # 2. Decode labels (handling -100 masking)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    # 3. Post-process: Fix gaps so <gap> matches ...\n    # This is CRITICAL for your score\n    decoded_preds = [replace_gaps_back(p.strip()) for p in decoded_preds]\n    decoded_labels = [replace_gaps_back(l.strip()) for l in decoded_labels]\n    \n    # 4. Debug: Print first example to see if it's garbage\n    # (Uncomment the next line if score stays low)\n    # print(f\"Sample: {decoded_preds[0]} || Ref: {decoded_labels[0]}\")\n    \n    # 5. Compute BLEU using SacreBLEU (Standard for MT)\n    # It handles short sentences much better than NLTK\n    bleu_output = sacrebleu.corpus_bleu(decoded_preds, [decoded_labels])\n    \n    # Return the score (0-100 scale)\n    return {\"bleu4\": bleu_output.score}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.910581Z","iopub.execute_input":"2026-01-15T14:29:29.910846Z","iopub.status.idle":"2026-01-15T14:29:29.984292Z","shell.execute_reply.started":"2026-01-15T14:29:29.910823Z","shell.execute_reply":"2026-01-15T14:29:29.983600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We need to load a model instance so the DataCollator knows how to \n# prepare the inputs (shift labels for decoder_input_ids)\nprint(\"Loading temporary model for DataCollator setup...\")\ntemp_model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer, \n    model=temp_model,  # <--- CRITICAL: Must pass model here\n    padding=True, \n    return_tensors=\"pt\"\n)\n\nprint(\"DataCollator ready with model-aware padding.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:29.985104Z","iopub.execute_input":"2026-01-15T14:29:29.985654Z","iopub.status.idle":"2026-01-15T14:29:40.270743Z","shell.execute_reply.started":"2026-01-15T14:29:29.985628Z","shell.execute_reply":"2026-01-15T14:29:40.270064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/input/byt5-small/pytorch/default/1/byt5-small\"\n\ndef objective(trial):\n    # 1. Hyperparameters to tune\n    # We lowered the max LR to prevent NaN loss\n    lr = trial.suggest_float(\"lr\", 1e-5, 3e-4, log=True)\n    epochs = trial.suggest_int(\"epochs\", 3, 7) # Reduced max epochs to save time\n    batch_size = trial.suggest_categorical(\"batch_size\", [4, 8]) \n    grad_acc = trial.suggest_categorical(\"grad_acc\", [4, 8]) \n    \n        # Safe subset selection\n    # Take 1000 OR the full dataset size, whichever is smaller\n    train_subset_size = min(1000, len(train_ds))\n    val_subset_size = min(200, len(val_ds))\n    \n    train_subset = train_ds.shuffle(seed=trial.number).select(range(train_subset_size))\n    val_subset = val_ds.shuffle(seed=trial.number).select(range(val_subset_size))\n\n    \n    # Check frequently (every 100 steps) so we can prune bad trials early\n    eval_steps = 100\n    save_steps = eval_steps * 2 # Must be multiple\n    \n    # 3. Model Setup\n    model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\n    \n    args = Seq2SeqTrainingArguments(\n        output_dir=f\"/tmp/trial_{trial.number}\",\n        overwrite_output_dir=True,\n        \n        # Core Params\n        per_device_train_batch_size=batch_size,\n        per_device_eval_batch_size=batch_size * 2,\n        gradient_accumulation_steps=grad_acc,\n        learning_rate=lr,\n        num_train_epochs=epochs,\n        \n        # Evaluation & Saving\n        eval_steps=eval_steps,\n        save_steps=save_steps,\n        logging_steps=50,\n        eval_strategy=\"steps\",\n        save_strategy=\"steps\",\n        load_best_model_at_end=True,\n        metric_for_best_model=\"bleu4\", # Optimizing for BLEU\n        greater_is_better=True,\n        \n        # Regularization\n        label_smoothing_factor=0.1, # Fixed to 0.1 for stability\n        warmup_ratio=0.1,\n        \n        # Generation (Critical for BLEU)\n        predict_with_generate=True,\n        generation_max_length=128,\n        generation_num_beams=2, # Keep low (2) for speed during tuning\n        \n        # Speed & Memory\n        fp16=True, # Enable for T4 speed\n        gradient_checkpointing=True, # Saves memory\n        dataloader_num_workers=0, # PREVENTS FREEZING\n        dataloader_pin_memory=False,\n        \n        # Cleanup\n        report_to=None,\n        save_total_limit=1,\n        ddp_find_unused_parameters=False,\n    )\n    \n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=args,\n        train_dataset=train_subset,\n        eval_dataset=val_subset,\n        tokenizer=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics,\n        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)] # Stop if not improving\n    )\n    \n    # Run Training\n    trainer.train()\n    eval_result = trainer.evaluate()\n    \n    # Cleanup to save GPU memory\n    del model, trainer, train_subset, val_subset\n    gc.collect()\n    torch.cuda.empty_cache()\n    \n    return eval_result[\"eval_bleu4\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:40.272839Z","iopub.execute_input":"2026-01-15T14:29:40.273093Z","iopub.status.idle":"2026-01-15T14:29:40.281615Z","shell.execute_reply.started":"2026-01-15T14:29:40.273071Z","shell.execute_reply":"2026-01-15T14:29:40.280939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import logging\nimport sys\n\n# Set Optuna to show every trial update\noptuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))\noptuna.logging.set_verbosity(optuna.logging.INFO)\n\n# Force Transformers to print progress\nfrom transformers.utils import logging as hf_logging\nhf_logging.set_verbosity_info()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:40.282710Z","iopub.execute_input":"2026-01-15T14:29:40.283019Z","iopub.status.idle":"2026-01-15T14:29:40.301192Z","shell.execute_reply.started":"2026-01-15T14:29:40.282998Z","shell.execute_reply":"2026-01-15T14:29:40.300599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TEMPORARY TEST: Tiny subset to verify it runs\ntrain_subset_size = 50  # Was min(1500, ...)\nval_subset_size = 10    # Was min(300, ...)\n\n# TEMPORARY TEST: Run only 1 epoch\nnum_train_epochs = 1   # Was trial.suggest_int(...)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:40.301878Z","iopub.execute_input":"2026-01-15T14:29:40.302272Z","iopub.status.idle":"2026-01-15T14:29:40.314477Z","shell.execute_reply.started":"2026-01-15T14:29:40.302238Z","shell.execute_reply":"2026-01-15T14:29:40.313973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use Hyperband to kill bad trials quickly (e.g. after 1 epoch)\nstudy = optuna.create_study(\n    direction=\"maximize\",\n    pruner=optuna.pruners.HyperbandPruner(\n        min_resource=1, \n        max_resource=7, \n        reduction_factor=3\n    )\n)\n\n# Run for 30 trials (good balance of time vs results)\n# Timeout set to 3 hours just in case\nstudy.optimize(objective, n_trials=30, timeout=10800, show_progress_bar=True)\n\nprint(\"=\"*50)\nprint(f\"BEST BLEU4: {study.best_value:.4f}\")\nprint(\"BEST PARAMS:\", study.best_params)\nprint(\"=\"*50)\n\n# Save\nstudy.trials_dataframe().to_csv(\"/kaggle/working/optuna_trials.csv\", index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T14:29:40.315319Z","iopub.execute_input":"2026-01-15T14:29:40.315575Z","iopub.status.idle":"2026-01-15T15:28:05.215823Z","shell.execute_reply.started":"2026-01-15T14:29:40.315530Z","shell.execute_reply":"2026-01-15T15:28:05.214375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna.visualization as vis\nimport pandas as pd\n\n# 1. Print Best Results clearly\nprint(\"=\"*50)\nprint(f\"BEST TRIAL ID: {study.best_trial.number}\")\nprint(f\"BEST BLEU4:    {study.best_value:.6f}\")\nprint(\"=\"*50)\nprint(\"BEST HYPERPARAMETERS:\")\nfor key, value in study.best_params.items():\n    print(f\"  {key:<20} : {value}\")\nprint(\"=\"*50)\n\n# 2. Create a DataFrame of all trials\ntrials_df = study.trials_dataframe()\n\n# Clean up column names for readability (remove 'params_' prefix)\ntrials_df.columns = [col.replace('params_', '') for col in trials_df.columns]\n\n# Sort by BLEU score (descending) to see top performers\ntop_trials = trials_df.sort_values(by='value', ascending=False).head(10)\n\nprint(\"\\nTOP 10 TRIALS:\")\n# Select relevant columns to display\ndisplay_cols = ['number', 'value', 'state', 'duration'] + list(study.best_params.keys())\nprint(top_trials[display_cols].to_markdown(index=False, numalign=\"left\", stralign=\"left\"))\n\n# 3. Save full details to CSV (useful to download and analyze locally)\ntrials_df.to_csv(\"optuna_detailed_results.csv\", index=False)\nprint(\"\\nSaved full trial history to 'optuna_detailed_results.csv'\")\n\n# 4. Visualizations (Interactive Plotly charts)\n# Note: These render directly in Kaggle notebooks\n\nprint(\"\\n1. HYPERPARAMETER IMPORTANCE (What mattered most?)\")\n# This tells you which parameter had the biggest impact on BLEU score\ntry:\n    fig_imp = vis.plot_param_importances(study)\n    fig_imp.show()\nexcept Exception as e:\n    print(\"Could not plot importance (need >1 completed trial).\")\n\nprint(\"\\n2. OPTIMIZATION HISTORY (Did we improve?)\")\n# This shows if the model got better over time\nfig_hist = vis.plot_optimization_history(study)\nfig_hist.show()\n\nprint(\"\\n3. SLICE PLOT (Where is the sweet spot?)\")\n# This shows individual dots for every trial for each parameter\nfig_slice = vis.plot_slice(study)\nfig_slice.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-15T15:28:05.216756Z","iopub.status.idle":"2026-01-15T15:28:05.217030Z","shell.execute_reply.started":"2026-01-15T15:28:05.216905Z","shell.execute_reply":"2026-01-15T15:28:05.216925Z"}},"outputs":[],"execution_count":null}]}