{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14225947,"sourceType":"datasetVersion","datasetId":9075299}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DEEP PAST CHALLENGE - BY JEMIRI DANIEL\n## Old Assyrian Cuneiform → English Neural Machine Translation V5\n## ByT5-BASE TRAINING—ENHANCED STRATEGY","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport torch\n\n# Memory optimization\nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n    torch.cuda.reset_peak_memory_stats()\n\nimport pandas as pd\nimport numpy as np\nfrom transformers import (\n    AutoModelForSeq2SeqLM,\n    AutoTokenizer,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n    DataCollatorForSeq2Seq\n)\nfrom datasets import Dataset as HFDataset\nimport re\nimport unicodedata\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\"*80)\nprint(\"mT5-SMALL FAST TRAINING - PRODUCTION READY\")\nprint(\"=\"*80)\nprint(f\"GPU: {torch.cuda.get_device_name(0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:03.966112Z","iopub.execute_input":"2026-01-06T10:50:03.966669Z","iopub.status.idle":"2026-01-06T10:50:04.669189Z","shell.execute_reply.started":"2026-01-06T10:50:03.966641Z","shell.execute_reply":"2026-01-06T10:50:04.668579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    DATA_DIR = '/kaggle/input/deep-past-initiative-machine-translation'\n    OUTPUT_DIR = '/kaggle/working/mt5_small_final'\n    \n    MODEL_NAME = \"/kaggle/input/mt5-small-complete/mt5-small-huggingface\"\n    \n    # Balanced speed & quality\n    MAX_SOURCE_LENGTH = 150\n    MAX_TARGET_LENGTH = 150\n    BATCH_SIZE = 2\n    GRADIENT_ACCUMULATION = 8      # Effective=16\n    LEARNING_RATE = 5e-5\n    WEIGHT_DECAY = 0.01\n    NUM_EPOCHS = 8\n    WARMUP_STEPS = 200\n    LABEL_SMOOTHING = 0.0\n    \n    EVAL_STEPS = 200\n    SAVE_STEPS = 200\n    LOGGING_STEPS = 25\n    \n    FP16 = True\n    GRADIENT_CHECKPOINTING = True\n    \n    # Speed boost: sample training data\n    SAMPLE_RATIO = 0.65            # Use 65% of data\n    \n    USE_GPU = torch.cuda.is_available()\n    SEED = 42\n\nconfig = Config()\ntorch.manual_seed(config.SEED)\nnp.random.seed(config.SEED)\n\nprint(f\"\\nConfiguration:\")\nprint(f\"  Max length: {config.MAX_SOURCE_LENGTH}\")\nprint(f\"  Batch: {config.BATCH_SIZE} x {config.GRADIENT_ACCUMULATION} = {config.BATCH_SIZE * config.GRADIENT_ACCUMULATION}\")\nprint(f\"  Epochs: {config.NUM_EPOCHS}\")\nprint(f\"  Data sampling: {config.SAMPLE_RATIO*100}%\")\nprint(f\"  Expected time: 3-4 hours\")\nprint(f\"  Target score: 3.5-4.5\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:04.670311Z","iopub.execute_input":"2026-01-06T10:50:04.670533Z","iopub.status.idle":"2026-01-06T10:50:04.677791Z","shell.execute_reply.started":"2026-01-06T10:50:04.670513Z","shell.execute_reply":"2026-01-06T10:50:04.677180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nLoading data...\")\ntrain_df = pd.read_csv(os.path.join(config.DATA_DIR, 'train.csv'))\nprint(f\"✓ Loaded {len(train_df)} documents\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:04.678716Z","iopub.execute_input":"2026-01-06T10:50:04.678967Z","iopub.status.idle":"2026-01-06T10:50:04.719661Z","shell.execute_reply.started":"2026-01-06T10:50:04.678940Z","shell.execute_reply":"2026-01-06T10:50:04.719000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    \"\"\"Fast cleaning\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = unicodedata.normalize('NFC', str(text))\n    text = ' '.join(text.split())\n    return text.strip()\n\nprint(\"\\nPreprocessing...\")\ntrain_df['transliteration'] = train_df['transliteration'].apply(clean_text)\ntrain_df['translation'] = train_df['translation'].apply(clean_text)\n\n# Remove invalid\ntrain_df = train_df[\n    (train_df['transliteration'].str.len() > 10) & \n    (train_df['translation'].str.len() > 10)\n].reset_index(drop=True)\n\n# Simple augmentation: only split very long documents\naugmented_data = []\n\nfor idx, row in train_df.iterrows():\n    translit = row['transliteration']\n    translation = row['translation']\n    \n    # Original\n    augmented_data.append({'transliteration': translit, 'translation': translation})\n    \n    # Only split if very long (>100 words)\n    translit_words = translit.split()\n    trans_words = translation.split()\n    \n    if len(translit_words) > 100:\n        mid_t = len(translit_words) // 2\n        mid_e = len(trans_words) // 2\n        \n        augmented_data.append({\n            'transliteration': ' '.join(translit_words[:mid_t]),\n            'translation': ' '.join(trans_words[:mid_e])\n        })\n        augmented_data.append({\n            'transliteration': ' '.join(translit_words[mid_t:]),\n            'translation': ' '.join(trans_words[mid_e:])\n        })\n\ntrain_df_final = pd.DataFrame(augmented_data).drop_duplicates().reset_index(drop=True)\n\nprint(f\"✓ Augmented:\")\nprint(f\"  Original: {len(train_df)}\")\nprint(f\"  Final: {len(train_df_final)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:04.720560Z","iopub.execute_input":"2026-01-06T10:50:04.720882Z","iopub.status.idle":"2026-01-06T10:50:04.869924Z","shell.execute_reply.started":"2026-01-06T10:50:04.720856Z","shell.execute_reply":"2026-01-06T10:50:04.869233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split\ntrain_size = int(0.92 * len(train_df_final))\ntrain_data = train_df_final[:train_size]\nval_data = train_df_final[train_size:]\n\n# SPEED BOOST: Sample training data\nsample_size = int(config.SAMPLE_RATIO * len(train_data))\ntrain_data = train_data.sample(n=sample_size, random_state=config.SEED).reset_index(drop=True)\n\nprint(f\"\\nDataset split:\")\nprint(f\"  Training: {len(train_data)} pairs ({config.SAMPLE_RATIO*100}% sample)\")\nprint(f\"  Validation: {len(val_data)} pairs\")\nprint(f\"  Speed improvement: ~{1/config.SAMPLE_RATIO:.1f}x faster\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:04.871821Z","iopub.execute_input":"2026-01-06T10:50:04.872058Z","iopub.status.idle":"2026-01-06T10:50:04.879868Z","shell.execute_reply.started":"2026-01-06T10:50:04.872039Z","shell.execute_reply":"2026-01-06T10:50:04.879168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nLoading model...\")\n\nmodel_path = \"/kaggle/input/mt5-small-complete/mt5-small-huggingface\"\n\ntokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n\nprint(f\"✓ Model: {model.num_parameters()/1e6:.0f}M parameters\")\n\n# Enable gradient checkpointing\nif config.GRADIENT_CHECKPOINTING:\n    model.gradient_checkpointing_enable()\n    print(\"✓ Gradient checkpointing enabled\")\n\nif config.USE_GPU:\n    model = model.cuda()\n    torch.cuda.empty_cache()\n    print(\"✓ Model on GPU\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:04.880903Z","iopub.execute_input":"2026-01-06T10:50:04.881261Z","iopub.status.idle":"2026-01-06T10:50:07.365263Z","shell.execute_reply.started":"2026-01-06T10:50:04.881234Z","shell.execute_reply":"2026-01-06T10:50:07.364518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTokenizing...\")\n\ndef preprocess_function(examples):\n    inputs = [\"translate Akkadian to English: \" + text for text in examples['transliteration']]\n    \n    model_inputs = tokenizer(\n        inputs,\n        max_length=config.MAX_SOURCE_LENGTH,\n        truncation=True,\n        padding=False\n    )\n    \n    labels = tokenizer(\n        examples['translation'],\n        max_length=config.MAX_TARGET_LENGTH,\n        truncation=True,\n        padding=False\n    )\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntrain_dataset = HFDataset.from_pandas(train_data)\nval_dataset = HFDataset.from_pandas(val_data)\n\ntrain_tokenized = train_dataset.map(\n    preprocess_function,\n    batched=True,\n    remove_columns=train_dataset.column_names\n)\n\nval_tokenized = val_dataset.map(\n    preprocess_function,\n    batched=True,\n    remove_columns=val_dataset.column_names\n)\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, padding=True)\n\nprint(f\"✓ Tokenized {len(train_tokenized)} training examples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:07.366291Z","iopub.execute_input":"2026-01-06T10:50:07.366755Z","iopub.status.idle":"2026-01-06T10:50:08.591145Z","shell.execute_reply.started":"2026-01-06T10:50:07.366731Z","shell.execute_reply":"2026-01-06T10:50:08.590536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir=config.OUTPUT_DIR,\n    num_train_epochs=config.NUM_EPOCHS,\n    per_device_train_batch_size=config.BATCH_SIZE,\n    per_device_eval_batch_size=config.BATCH_SIZE,\n    learning_rate=config.LEARNING_RATE,\n    weight_decay=config.WEIGHT_DECAY,\n    warmup_steps=config.WARMUP_STEPS,\n    label_smoothing_factor=config.LABEL_SMOOTHING,\n    gradient_accumulation_steps=config.GRADIENT_ACCUMULATION,\n    gradient_checkpointing=config.GRADIENT_CHECKPOINTING,\n    fp16=config.FP16,\n    eval_strategy=\"steps\",\n    eval_steps=config.EVAL_STEPS,\n    save_strategy=\"steps\",\n    save_steps=config.SAVE_STEPS,\n    logging_steps=config.LOGGING_STEPS,\n    save_total_limit=2,\n    load_best_model_at_end=False,\n    report_to=\"none\",\n    seed=config.SEED,\n    dataloader_num_workers=0,\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_tokenized,\n    eval_dataset=val_tokenized,\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n)\n\nprint(\"✓ Trainer configured\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:08.591926Z","iopub.execute_input":"2026-01-06T10:50:08.592190Z","iopub.status.idle":"2026-01-06T10:50:08.798362Z","shell.execute_reply.started":"2026-01-06T10:50:08.592170Z","shell.execute_reply":"2026-01-06T10:50:08.797630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TRAINING STARTED\")\nprint(\"=\"*80)\nprint(f\"Dataset: {len(train_tokenized)} pairs\")\nprint(f\"Epochs: {config.NUM_EPOCHS}\")\nprint(f\"Steps per epoch: ~{len(train_tokenized) // (config.BATCH_SIZE * config.GRADIENT_ACCUMULATION)}\")\nprint(f\"Total steps: ~{(len(train_tokenized) // (config.BATCH_SIZE * config.GRADIENT_ACCUMULATION)) * config.NUM_EPOCHS}\")\nprint(f\"Expected time: 3-4 hours\")\nprint(\"=\"*80 + \"\\n\")\n\ntry:\n    train_result = trainer.train()\n    print(\"\\n\" + \"=\"*80)\n    print(\"✓ TRAINING COMPLETE!\")\n    print(\"=\"*80)\nexcept Exception as e:\n    print(f\"\\n⚠ Training interrupted: {e}\")\n    print(\"Saving current checkpoint...\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:50:08.799528Z","iopub.execute_input":"2026-01-06T10:50:08.799754Z","execution_failed":"2026-01-06T13:07:28.238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nSaving model...\")\nmodel.save_pretrained(config.OUTPUT_DIR)\ntokenizer.save_pretrained(config.OUTPUT_DIR)\nprint(f\"✓ Model saved to: {config.OUTPUT_DIR}\")\n\n# Save training info\ntraining_info = {\n    'model': 'mt5-small',\n    'epochs': config.NUM_EPOCHS,\n    'max_length': config.MAX_SOURCE_LENGTH,\n    'batch_size': config.BATCH_SIZE * config.GRADIENT_ACCUMULATION,\n    'training_samples': len(train_tokenized),\n}\n\nimport json\nwith open(os.path.join(config.OUTPUT_DIR, 'training_info.json'), 'w') as f:\n    json.dump(training_info, f, indent=2)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"NEXT STEPS\")\nprint(\"=\"*80)\nprint(\"1. After training completes, click 'Save Version'\")\nprint(\"2. Go to Output → Save as Dataset → Name: 'mt5-small-akkadian-final'\")\nprint(\"3. Use this dataset in your inference notebook\")\nprint(f\"\\nExpected score: 3.5-4.5\")\nprint(\"=\"*80)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-06T13:07:28.239Z"}},"outputs":[],"execution_count":null}]}