{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "a3uc814g1oW6", "outputId": "945c9f43-ee11-43eb-9a2b-e30d7c145d88" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mounted at /content/drive\n" ] } ], "source": [ "from google.colab import drive\n", "drive.mount('/content/drive', force_remount=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SL-xpVQ21ohw" }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T19:59:53.180660Z", "iopub.status.busy": "2026-03-10T19:59:53.180309Z", "iopub.status.idle": "2026-03-10T20:00:18.203907Z", "shell.execute_reply": "2026-03-10T20:00:18.203013Z", "shell.execute_reply.started": "2026-03-10T19:59:53.180631Z" }, "id": "xEg13TEPCE6c" }, "outputs": [], "source": [ "!pip install sacrebleu\n", "!pip install --upgrade wandb\n", "!pip install transformers==4.57.1\n", "!pip install evaluate" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6_A887fnh32s" }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-01-17T19:06:12.052894Z", "iopub.status.busy": "2026-01-17T19:06:12.052555Z", "iopub.status.idle": "2026-01-17T19:06:12.057642Z", "shell.execute_reply": "2026-01-17T19:06:12.056676Z", "shell.execute_reply.started": "2026-01-17T19:06:12.052861Z" }, "id": "QgmMwzEfdfDX" }, "outputs": [], "source": [ "# !pip install -q -U bitsandbytes\n", "# !pip install -q -U accelerate\n", "# !pip install -q -U peft" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T20:00:18.206275Z", "iopub.status.busy": "2026-03-10T20:00:18.205938Z", "iopub.status.idle": "2026-03-10T20:00:53.493725Z", "shell.execute_reply": "2026-03-10T20:00:53.492923Z", "shell.execute_reply.started": "2026-03-10T20:00:18.206231Z" }, "id": "qa3xe4xo52p1" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import torch\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "import sacrebleu\n", "import math\n", "import re\n", "import os\n", "# import wandb\n", "# from kaggle_secrets import UserSecretsClient\n", "from transformers import TrainerCallback\n", "from pathlib import Path\n", "import shutil\n", "\n", "from datasets import load_dataset\n", "\n", "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, Seq2SeqTrainer, Seq2SeqTrainingArguments#, BitsAndBytesConfig\n", "# from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model, TaskType\n", "\n", "from torch.cuda import temperature" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T20:00:53.496416Z", "iopub.status.busy": "2026-03-10T20:00:53.495273Z", "iopub.status.idle": "2026-03-10T20:00:53.502029Z", "shell.execute_reply": "2026-03-10T20:00:53.500494Z", "shell.execute_reply.started": "2026-03-10T20:00:53.496360Z" }, "id": "RxOYzwpQDIkK" }, "outputs": [], "source": [ "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mADjUBA5Orcl" }, "outputs": [], "source": [ "# Fix the seed (for reproducibility).\n", "def seed_everything(seed=42):\n", " torch.manual_seed(seed)\n", " np.random.seed(seed)\n", " torch.cuda.manual_seed(seed)\n", "\n", "seed_everything()" ] }, { "cell_type": "markdown", "metadata": { "id": "hW6QEuysCPQY" }, "source": [ "# Train Single Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zAHheR2HPKyz" }, "outputs": [], "source": [ "class Config:\n", " OUTPUT_DIR = \"/content/drive/MyDrive/Deep Past Challenge 26/models/aduah_LARGE_from_zero_v8\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "442764af8f8a4dac8c07d393d3c82ce5", "5d012c0db9294a27941c19cc5fb0ce87", "aebbd8bf99a74e888d69b67ce18ae2fd", "4bf6b555de3349eead9f98c4d9812ade", "b197095895d94eabb700b11baf605150", "c0a4bb18c47343858e20b254645af31e", "2447d668ca9442cab65748241ed0e599", "1021e9c96c194eedb6c7901a1060fa9a", "f655d61c188b4bbeae62913fa65acd3f", "b73a157609014218bf7c8662b947ae48", "51a10dae678e4275a102f934e79e947f", "64374a975316404a9e9c2cf2e23f78c7", "381f6854e0e24e7cade22e1ff36fa909", "6fa46b7b71e64f689c51432926595f7d", "77a1fd81e0b24044be73920f270d3bbb", "a74062b71d9946c5b3d0a3673bb30189", "9c81f86549e64298b78dc87d809a2cf7", "ab7469ad48734b028f8fcc66be1c9d3e", "cc612ac15fc14d308b3577b98e267e37", "6b32213615604c70900d4fcb0a502a6f", "c0582acce6ad41de94e397360a31381a", "f4fe98cad5614ad781b2dbb04ff242a4", "7b263e0b646b4c23929bf2f3ccb17f0f", "47e04d0f760c44d9853a000ea4dd13c6", "2fa323d8516443ba95dee8568d7a5507", "406f60f150094e0bae1f40a372c09f38", "32ec4e8d0afe4fbeb9f13a2fbce0fc36", "672cb05b6a68423b9852bfc48e89f3f8", "f38c3aac4cad41c29d651e8d8dbdc6b6", "a314a9f4adf041dfaa9e91db3d86f089", "291ba9ac20954b1098238bdbfae07b7a", "c027ecf1f3d14d22880950f06c512521", "bad71dc8623a4ff6a2795f1b403c8513", "ec450bf5d02f4e8aa2d24545aaf46a22", "799039598d4f495c8ed32a5a9f9aed0e", "a65568861c604d8489f9d07cdbbff989", "34f5629d605b4cac8a18bf2d281b6ffa", "cee79010bfcd4feabd50ba0ae1bde883", "a87ec5c8faa849cc97028fce837b3bb0", "f5ec8602d950470b97b95684ce31249c", "edf3aef1eaa24226b41c67327a988a7b", "729481699bfa48dd8b14b7095aa346c7", "ce76bf34834449d182e5bf7811ec98a1", "7b38d236ec7f4db28346d090a8f0629f", "a7cc5e15199a41b8a8b9f7e9df61fd47", "9b9c717f133e4a0b986919dd860835da", "53d37c0563194745b6bf5e4799da3d57", "c2d5fccd2cd942e6b50b33a8ce30d488", "d5721ba5411b4c4ca420438b00497ffa", "3001b1aaace048f98e31c72a89a4fab7", "8fa40474b6694352bed3a7b5e4d029f5", "b1d420ace46f4faab9a8297a04c4506f", "03ad04511668436195a1da342a99204b", "81a0c7dd1cac4bdba26c192bcf9bc43f", "70f84f3c90f446eea016d8530a324f77", "d1e205b52462483997a7f94df4eb8b99", "385ebb06237b4f87a6a0c3142af56e95", "1f60cc80cc824eaeaad4bbb4dbe9c67d", "95ab004961e247289a84037989f5f0e9", "c934be2ff1a548d4a8e0265b6d08ea99", "aab385a8b64d4864b41ec2752000aa83", "5474e804158c4191a9c9d18b6a77bcd3", "4cce577f6272406588b8b3da6434f1ad", "67943366c4d448de9a8468f9fc54e498", "e0d4dd064a6448889bd2a986c5e84821", "ab58473e245845dfbdb5386cfa52d9c7", "1cb6e142da43445ba0ef6a07bcb8eda9", "136803f62009480e9939a1a259f8c015", "5c49488c516d400c82e8d6a9b81ffecf", "d63cdd1f6a1044fd8a9fbf23d5e7e76a", "8605e90a09a943b493552b06ecbb9700", "fbb0ce59601c41cc95c7aa08d29985c1", "f97c24e253df40de82d8fc2fae4527d0", "812f32df56af48c188538e1ed3b393c4", "beb12a7b73324e6e99ec557bb81556fb", "0c071c10324642f38921542351774057", "95105126e89d44aaa23c51c75ea9e016" ] }, "id": "Kqjw1hQCRNb6", "outputId": "ca7dbe80-e40e-4826-f49f-ab9bd5668e23" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "442764af8f8a4dac8c07d393d3c82ce5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "config.json: 0%| | 0.00/642 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n", "/usr/local/lib/python3.12/dist-packages/transformers/convert_slow_tokenizer.py:564: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n", " warnings.warn(\n" ] }, { "data": { "text/plain": [ "MT5ForConditionalGeneration(\n", " (shared): Embedding(250112, 1024)\n", " (encoder): MT5Stack(\n", " (embed_tokens): Embedding(250112, 1024)\n", " (block): ModuleList(\n", " (0): MT5Block(\n", " (layer): ModuleList(\n", " (0): MT5LayerSelfAttention(\n", " (SelfAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " (relative_attention_bias): Embedding(32, 16)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): MT5LayerFF(\n", " (DenseReluDense): MT5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (1-23): 23 x MT5Block(\n", " (layer): ModuleList(\n", " (0): MT5LayerSelfAttention(\n", " (SelfAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): MT5LayerFF(\n", " (DenseReluDense): MT5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (final_layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (decoder): MT5Stack(\n", " (embed_tokens): Embedding(250112, 1024)\n", " (block): ModuleList(\n", " (0): MT5Block(\n", " (layer): ModuleList(\n", " (0): MT5LayerSelfAttention(\n", " (SelfAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " (relative_attention_bias): Embedding(32, 16)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): MT5LayerCrossAttention(\n", " (EncDecAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): MT5LayerFF(\n", " (DenseReluDense): MT5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (1-23): 23 x MT5Block(\n", " (layer): ModuleList(\n", " (0): MT5LayerSelfAttention(\n", " (SelfAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): MT5LayerCrossAttention(\n", " (EncDecAttention): MT5Attention(\n", " (q): Linear(in_features=1024, out_features=1024, bias=False)\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(in_features=1024, out_features=1024, bias=False)\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): MT5LayerFF(\n", " (DenseReluDense): MT5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (final_layer_norm): MT5LayerNorm()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (lm_head): Linear(in_features=1024, out_features=250112, bias=False)\n", ")" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from transformers import AutoModelForSeq2SeqLM, AutoConfig, AutoTokenizer\n", "\n", "model = AutoModelForSeq2SeqLM.from_pretrained(\"google/mt5-large\")\n", "tokenizer = AutoTokenizer.from_pretrained(\"google/mt5-large\")\n", "model.to(device)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "td0yOwHr6TQz" }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2QPPrBp8Lk_v" }, "outputs": [], "source": [ "model.generation_config.num_beams = 5\n", "model.generation_config.length_penalty = 1.7\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T05:39:20.918526Z", "iopub.status.busy": "2026-03-10T05:39:20.918015Z", "iopub.status.idle": "2026-03-10T05:39:21.611886Z", "shell.execute_reply": "2026-03-10T05:39:21.611210Z", "shell.execute_reply.started": "2026-03-10T05:39:20.918480Z" }, "id": "IQoSb-Le6z29" }, "outputs": [], "source": [ "dataset_directory = \"/content/drive/MyDrive/Deep Past Challenge 26/data\"\n", "dataset = load_dataset(\"csv\", data_files={\"train\": f\"{dataset_directory}/train_dataset.csv\", \"validation\": f\"{dataset_directory}/val_set.csv\"})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T05:39:21.613373Z", "iopub.status.busy": "2026-03-10T05:39:21.613052Z", "iopub.status.idle": "2026-03-10T05:39:21.618446Z", "shell.execute_reply": "2026-03-10T05:39:21.617691Z", "shell.execute_reply.started": "2026-03-10T05:39:21.613340Z" }, "id": "2Tmu4r2v-IgC" }, "outputs": [], "source": [ "MAX_SOURCE_LEN = 600\n", "MAX_TARGET_LEN = 690" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2026-03-10T20:01:44.399982Z", "iopub.status.busy": "2026-03-10T20:01:44.399655Z", "iopub.status.idle": "2026-03-10T20:01:44.406382Z", "shell.execute_reply": "2026-03-10T20:01:44.405291Z", "shell.execute_reply.started": "2026-03-10T20:01:44.399951Z" }, "id": "Y4Z_h5zy-VTf" }, "outputs": [], "source": [ "def preprocess(batch):\n", " inputs = tokenizer(\n", " batch[\"transliteration\"],\n", " max_length=MAX_SOURCE_LEN,\n", " padding=\"max_length\",\n", " truncation=True,\n", " )\n", "\n", " # ByT5 does not need as_target_tokenizer()\n", " labels = tokenizer(\n", " batch[\"translation\"],\n", " max_length=MAX_TARGET_LEN,\n", " padding=\"max_length\",\n", " truncation=True,\n", " )\n", "\n", " # CRITICAL LINE\n", " labels[\"input_ids\"] = [\n", " [(token if token != tokenizer.pad_token_id else -100) for token in seq]\n", " for seq in labels[\"input_ids\"]\n", " ]\n", "\n", " inputs[\"labels\"] = labels[\"input_ids\"]\n", " return inputs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 81, "referenced_widgets": [ "e667c903b74d4c38ae2601b79a0d3214", "116ead90268f4a90aa73a51eb8122986", "539f0b91359648df99fb512d2a605b13", "5dc8900e3ecd44b197f1d2023de18284", "418ff96729e44ec482a9e7809d982578", "2b9cafdf5af04abc87505e7552f37466", "3548c83148774859b4b0d2191ed48934", "77b02b81d1fd4c4a90e55886cd53f196", "5fe11c19421542f1a367c19e1eca6924", "fa971499c07645b5967d6f6fa7cd0473", "d65a7fff843643dcbef674f60c76610d", "1143e996ace84b7280c3ebb52556c994", "fd532b943d054bcda15401c82ce8610c", "d29aea82752b445798c66012edf9c600", "c12d8437af474fa9b2128ccecdaab47b", "0062d4987c6e4f6ca03e4353c719e07b", "055c690e9f2b497d9b663f99e50b60ea", "ff545147d3384099990292b8e493335e", "983e4d1678c0476f88c0247012d0eff5", "a8d11fc0f5354dc5b68aed21b7b0544a", "2d6651d6bb5b4ef7ac9f1da145405d3c", "5ff08a5971e64cbc9a3566e24e16eacf" ] }, "execution": { "iopub.execute_input": "2026-03-10T05:39:21.635902Z", "iopub.status.busy": "2026-03-10T05:39:21.635692Z", "iopub.status.idle": "2026-03-10T05:40:05.338296Z", "shell.execute_reply": "2026-03-10T05:40:05.337658Z", "shell.execute_reply.started": "2026-03-10T05:39:21.635881Z" }, "id": "biF5hAM2_ctE", "outputId": "25c883d1-915d-4e8a-9532-afb5a21a6f13" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e667c903b74d4c38ae2601b79a0d3214", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/14532 [00:00\n", " \n", " \n", " [12532/33950 7:34:59 < 12:57:44, 0.46 it/s, Epoch 25.84/70]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossBleuChrf++Score
1No log0.6356738.30666426.56083514.853684
22.0310000.41664527.27112847.21756735.884207
30.5685000.34536334.93146954.82049443.760261
40.4069000.31897440.00209056.98753047.745369
50.3420000.30497940.57010058.76205748.826044
60.3020000.29673741.15521358.66023549.134249
70.2730000.29697842.12006758.69511049.721645
80.2478000.29311643.35050660.47483151.201704
90.2283000.29768444.45449961.13078552.130015
100.2090000.29924344.79977161.41905452.455310
110.1910000.31313141.78355658.78744549.561563
120.1746000.32088543.00798659.95609150.779826
130.1558000.32118742.80849959.87984950.629699
140.1430000.34351143.20391059.84843450.849644
150.1293000.34693941.76175758.97124949.626031
160.1171000.36137142.46783459.79676550.392848
170.1048000.37664141.81140758.98663349.661999
180.0944000.38219340.71980858.31600148.730036
190.0834000.39319141.84418059.16323349.755773
200.0766000.40944840.82906658.38577248.824548
210.0677000.43345841.50530458.54391549.293843
220.0607000.43439141.94868759.14212349.808979
230.0546000.45934140.37912958.28185648.511551
240.0499000.46399740.87522958.32295548.825855
250.0452000.47731641.71729958.92219149.578974

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/tmp/ipykernel_40972/1114521145.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 18\u001b[0m ]\n\u001b[1;32m 19\u001b[0m )\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m 2323\u001b[0m \u001b[0mhf_hub_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_progress_bars\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2324\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2325\u001b[0;31m return inner_training_loop(\n\u001b[0m\u001b[1;32m 2326\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2327\u001b[0m \u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 2677\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogging_nan_inf_filter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2678\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_torch_xla_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2679\u001b[0;31m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnan\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_loss_step\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misinf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_loss_step\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2680\u001b[0m ):\n\u001b[1;32m 2681\u001b[0m \u001b[0;31m# if loss is nan or inf simply add the average of previous logged losses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "from transformers import DataCollatorForSeq2Seq\n", "\n", "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n", "\n", "trainer = Seq2SeqTrainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"validation\"],\n", " data_collator=data_collator,\n", " compute_metrics=compute_metrics,\n", " callbacks=[\n", " KeepBestDeleteWorstCallback(),\n", " # SaveEvery5EpochsFrom20Callback( # ← добавляем\n", " # tokenizer=tokenizer,\n", " # save_dir=Config.OUTPUT_DIR\n", " # )\n", " ]\n", ")\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TyIyiwt0bMpa" }, "outputs": [], "source": [ "# --- Save Model ---\n", "# Important: the model saved here will be loaded in the next notebook.\n", "trainer.save_model(Config.OUTPUT_DIR)\n", "tokenizer.save_pretrained(Config.OUTPUT_DIR)\n", "print(f\"Model saved to {Config.OUTPUT_DIR}\")\n" ] }, { "cell_type": "markdown", "metadata": { "id": "1YndrtbCH6ym" }, "source": [ "# Average Checkpoints" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 347 }, "id": "-QHkaqVg2HnJ", "outputId": "077425bf-66b7-4712-cd29-d0c5223116c7" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Загружаем [1/2] weight=0.50: /content/drive/MyDrive/Deep Past Challenge 26/models/aduah_LARGE_from_zero_v8/checkpoint-4850\n", " VRAM свободно: 82.3GB\n", "Загружаем [2/2] weight=0.50: /content/drive/MyDrive/Deep Past Challenge 26/models/aduah_LARGE_from_zero_v7_continue_3/checkpoint-970\n", " VRAM свободно: 82.3GB\n", "Загружаем усреднённую модель...\n", "Оцениваем усреднённую модель...\n" ] }, { "data": { "text/html": [ "\n", "

\n", " \n", " \n", " [40/40 01:42]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Averaged (0.5/0.5): {'eval_loss': 0.3056432604789734, 'eval_model_preparation_time': 0.0062, 'eval_bleu': 45.39070959271189, 'eval_chrf++': 61.69822213987071, 'eval_score': 52.9199970099914, 'eval_runtime': 114.4694, 'eval_samples_per_second': 1.73, 'eval_steps_per_second': 0.349}\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [40/40 01:40]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Чекпоинт 1: {'eval_loss': 0.2992440164089203, 'eval_model_preparation_time': 0.0062, 'eval_bleu': 44.799770799631915, 'eval_chrf++': 61.41905440097537, 'eval_score': 52.4553101210337, 'eval_runtime': 111.7716, 'eval_samples_per_second': 1.771, 'eval_steps_per_second': 0.358}\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [40/40 01:43]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Чекпоинт 2: {'eval_loss': 0.3260941207408905, 'eval_model_preparation_time': 0.006, 'eval_bleu': 44.42105908313452, 'eval_chrf++': 61.230084720625065, 'eval_score': 52.15271048603532, 'eval_runtime': 114.4613, 'eval_samples_per_second': 1.73, 'eval_steps_per_second': 0.349}\n", "\n", "=== СРАВНЕНИЕ ===\n", "Чекпоинт 1 : score = 52.4553\n", "Чекпоинт 2 : score = 52.1527\n", "Averaged : score = 52.9200\n" ] } ], "source": [ "import gc\n", "from transformers import DataCollatorForSeq2Seq\n", "\n", "def average_checkpoints(checkpoint_paths: list, weights: list = None) -> AutoModelForSeq2SeqLM:\n", " n = len(checkpoint_paths)\n", " if weights is None:\n", " weights = [1.0 / n] * n\n", "\n", " avg_state_dict = {}\n", "\n", " for i, (ckpt_path, w) in enumerate(zip(checkpoint_paths, weights)):\n", " print(f\"Загружаем [{i+1}/{n}] weight={w:.2f}: {ckpt_path}\")\n", " model = AutoModelForSeq2SeqLM.from_pretrained(ckpt_path)\n", " state_dict = model.state_dict()\n", "\n", " for key in state_dict:\n", " if key not in avg_state_dict:\n", " avg_state_dict[key] = torch.zeros_like(state_dict[key]).float()\n", " avg_state_dict[key] += w * state_dict[key].float()\n", "\n", " del model\n", " gc.collect()\n", " torch.cuda.empty_cache()\n", " print(f\" VRAM свободно: {(torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated())/1e9:.1f}GB\")\n", "\n", " # Загружаем усреднённые веса\n", " print(\"Загружаем усреднённую модель...\")\n", " result_model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_paths[0])\n", " result_model.load_state_dict(avg_state_dict)\n", " return result_model\n", "\n", "\n", "# ---- Усредняем два чекпоинта ----\n", "ckpt_1 = \"/content/drive/MyDrive/Deep Past Challenge 26/models/aduah_LARGE_from_zero_v8/checkpoint-4850\"\n", "ckpt_2 = \"/content/drive/MyDrive/Deep Past Challenge 26/models/aduah_LARGE_from_zero_v7_continue_3/checkpoint-970\"\n", "\n", "averaged_model = average_checkpoints(\n", " checkpoint_paths=[ckpt_1, ckpt_2],\n", " weights=[0.5, 0.5]\n", ")\n", "tokenizer = AutoTokenizer.from_pretrained(ckpt_1)\n", "\n", "# ---- Оцениваем через trainer.evaluate() ----\n", "# model.generation_config.num_beams = 5\n", "# model.generation_config.length_penalty = 1.7\n", "\n", "training_args_eval = Seq2SeqTrainingArguments(\n", " output_dir=\"/tmp/eval_avg\",\n", " predict_with_generate=True,\n", " generation_max_length=690,\n", " per_device_eval_batch_size=5,\n", " fp16=False,\n", " report_to=\"none\",\n", " remove_unused_columns=False,\n", ")\n", "\n", "trainer_eval = Seq2SeqTrainer(\n", " model=averaged_model,\n", " args=training_args_eval,\n", " eval_dataset=dataset[\"validation\"],\n", " data_collator=DataCollatorForSeq2Seq(tokenizer, model=averaged_model),\n", " compute_metrics=compute_metrics,\n", ")\n", "trainer_eval.args.generation_num_beams = 5\n", "\n", "print(\"Оцениваем усреднённую модель...\")\n", "metrics_avg = trainer_eval.evaluate()\n", "print(f\"Averaged (0.5/0.5): {metrics_avg}\")\n", "\n", "# ---- Для сравнения — оцениваем каждый чекпоинт отдельно ----\n", "metrics_single_all = []\n", "for ckpt_path, label in [(ckpt_1, \"Чекпоинт 1\"), (ckpt_2, \"Чекпоинт 2\")]:\n", " model_single = AutoModelForSeq2SeqLM.from_pretrained(ckpt_path)\n", "\n", " trainer_single = Seq2SeqTrainer(\n", " model=model_single,\n", " args=training_args_eval,\n", " eval_dataset=dataset[\"validation\"],\n", " data_collator=DataCollatorForSeq2Seq(tokenizer, model=model_single),\n", " compute_metrics=compute_metrics,\n", " )\n", " trainer_single.args.generation_num_beams = 5\n", "\n", " metrics_single = trainer_single.evaluate()\n", " metrics_single_all.append(metrics_single)\n", " print(f\"{label}: {metrics_single}\")\n", "\n", " del model_single\n", " gc.collect()\n", " torch.cuda.empty_cache()\n", "\n", "# ---- Итоговое сравнение ----\n", "print(\"\\n=== СРАВНЕНИЕ ===\")\n", "print(f\"Чекпоинт 1 : score = {metrics_single_all[0]['eval_score']:.4f}\")\n", "print(f\"Чекпоинт 2 : score = {metrics_single_all[1]['eval_score']:.4f}\")\n", "print(f\"Averaged : score = {metrics_avg['eval_score']:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "sT7NoBPhN1dw", "outputId": "c8a0ebcc-d141-484d-b8cc-c6cfbd30c361" }, "outputs": [ { "data": { "text/plain": [ "('LARGE_MIXES_v1/tokenizer_config.json',\n", " 'LARGE_MIXES_v1/special_tokens_map.json',\n", " 'LARGE_MIXES_v1/added_tokens.json')" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "save_path = \"LARGE_MIXES_v1\"\n", "\n", "averaged_model.save_pretrained(save_path)\n", "tokenizer.save_pretrained(save_path)" ] }, { "cell_type": "markdown", "metadata": { "id": "rLQTkhFMPrUG" }, "source": [ "# Ensemble MBR" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OdA4VJiEZBWs" }, "outputs": [], "source": [ "\"\"\"\n", "Ensemble MBR Pipeline\n", "=====================\n", "Этапы:\n", " 1. Определение моделей (ModelConfig)\n", " 2. Генерация кандидатов от каждой модели последовательно\n", " 3. Сбор кандидатов в DataFrame\n", " 4. MBR decoding — выбор лучшего кандидата\n", " 5. (Опционально) Оценка на валидации\n", "\"\"\"\n", "\n", "import gc\n", "import math\n", "from dataclasses import dataclass, field\n", "from typing import List, Optional, Dict\n", "\n", "import pandas as pd\n", "import numpy as np\n", "import sacrebleu\n", "import torch\n", "from tqdm import tqdm\n", "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n", "\n", "\n", "# ============================================================\n", "# Этап 0: Конфигурация моделей\n", "# ============================================================\n", "\n", "@dataclass\n", "class ModelConfig:\n", " \"\"\"Описание одной модели для ensemble\"\"\"\n", " name: str # Человекочитаемое имя\n", " model_path: str # Путь к весам\n", " tokenizer_path: Optional[str] = None # Путь к токенайзеру (если отличается)\n", "\n", " # Generation параметры\n", " num_beams: int = 12\n", " num_return: int = 5\n", " max_input_length: int = 600\n", " max_output_length: int = 690\n", " length_penalty: float = 1.7\n", "\n", " # Batch\n", " batch_size: int = 5\n", "\n", " # Доп. параметры загрузки\n", " torch_dtype: Optional[torch.dtype] = None # например torch.bfloat16\n", " local_files_only: bool = True\n", "\n", " def __post_init__(self):\n", " if self.tokenizer_path is None:\n", " self.tokenizer_path = self.model_path\n", "\n", "\n", "# ============================================================\n", "# Этап 1: Генерация кандидатов от одной модели\n", "# ============================================================\n", "\n", "def generate_candidates_single_model(\n", " config: ModelConfig,\n", " texts: List[str],\n", " device: str = \"cuda\",\n", ") -> List[List[str]]:\n", " \"\"\"\n", " Загружает модель, генерирует кандидатов, освобождает память.\n", "\n", " Returns:\n", " List[List[str]]: для каждого текста — список из num_return кандидатов\n", " \"\"\"\n", " print(f\"\\n{'='*60}\")\n", " print(f\"[{config.name}] Loading model from {config.model_path}\")\n", " print(f\" beams={config.num_beams}, return={config.num_return}, \"\n", " f\"lp={config.length_penalty}, batch={config.batch_size}\")\n", " print(f\"{'='*60}\")\n", "\n", " # Загрузка\n", " load_kwargs = {\"local_files_only\": config.local_files_only}\n", " if config.torch_dtype is not None:\n", " load_kwargs[\"torch_dtype\"] = config.torch_dtype\n", "\n", " model = AutoModelForSeq2SeqLM.from_pretrained(config.model_path, **load_kwargs)\n", " model.to(device)\n", " model.eval()\n", "\n", " tokenizer = AutoTokenizer.from_pretrained(\n", " config.tokenizer_path, local_files_only=config.local_files_only\n", " )\n", "\n", " # Генерация\n", " all_candidates = []\n", " n_batches = math.ceil(len(texts) / config.batch_size)\n", "\n", " for i in tqdm(range(0, len(texts), config.batch_size),\n", " total=n_batches, desc=f\"[{config.name}] Generating\"):\n", " batch_texts = texts[i : i + config.batch_size]\n", "\n", " inputs = tokenizer(\n", " batch_texts,\n", " return_tensors=\"pt\",\n", " max_length=config.max_input_length,\n", " padding=True,\n", " truncation=True,\n", " ).to(device)\n", "\n", " with torch.no_grad():\n", " outputs = model.generate(\n", " **inputs,\n", " num_beams=config.num_beams,\n", " num_return_sequences=config.num_return,\n", " max_length=config.max_output_length,\n", " length_penalty=config.length_penalty,\n", " early_stopping=True,\n", " )\n", "\n", " decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n", "\n", " # Разбить по примерам: [batch_size * num_return] → batch_size × num_return\n", " for j in range(len(batch_texts)):\n", " start = j * config.num_return\n", " end = start + config.num_return\n", " cands = [d.strip() for d in decoded[start:end]]\n", " all_candidates.append(cands)\n", "\n", " # Освобождение памяти\n", " del model, tokenizer\n", " gc.collect()\n", " torch.cuda.empty_cache()\n", "\n", " mem_free = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated()\n", " print(f\"[{config.name}] Done. Generated {len(all_candidates)} × {config.num_return} candidates. \"\n", " f\"VRAM free: {mem_free / 1e9:.1f} GB\")\n", "\n", " return all_candidates\n", "\n", "\n", "# ============================================================\n", "# Этап 2: Прогон всех моделей, сбор кандидатов в DataFrame\n", "# ============================================================\n", "\n", "def generate_all_candidates(\n", " model_configs: List[ModelConfig],\n", " texts: List[str],\n", " device: str = \"cuda\",\n", ") -> pd.DataFrame:\n", " \"\"\"\n", " Последовательно прогоняет все модели.\n", "\n", " Returns:\n", " DataFrame с колонками:\n", " - text_idx: индекс текста\n", " - input_text: исходный текст\n", " - {model_name}_0, {model_name}_1, ...: кандидаты от каждой модели\n", " \"\"\"\n", " n = len(texts)\n", "\n", " # Базовый DataFrame\n", " df = pd.DataFrame({\n", " \"text_idx\": range(n),\n", " \"input_text\": texts,\n", " })\n", "\n", " # Прогон каждой модели\n", " for config in model_configs:\n", " candidates = generate_candidates_single_model(config, texts, device)\n", "\n", " # Добавить колонки: model_name_0, model_name_1, ...\n", " for k in range(config.num_return):\n", " col_name = f\"{config.name}_{k}\"\n", " df[col_name] = [candidates[i][k] if k < len(candidates[i]) else \"\"\n", " for i in range(n)]\n", "\n", " return df\n", "\n", "\n", "def get_candidate_columns(df: pd.DataFrame) -> List[str]:\n", " \"\"\"Вернуть имена колонок с кандидатами\"\"\"\n", " return [c for c in df.columns if c not in (\"text_idx\", \"input_text\")]\n", "\n", "\n", "def get_candidates_for_row(row: pd.Series, candidate_cols: List[str]) -> List[str]:\n", " \"\"\"Извлечь всех кандидатов для одного примера\"\"\"\n", " return [row[c] for c in candidate_cols if row[c] and str(row[c]).strip()]\n", "\n", "\n", "# ============================================================\n", "# Этап 3: MBR Decoding\n", "# ============================================================\n", "\n", "def mbr_decode_chrf(candidates: List[str]) -> str:\n", " \"\"\"MBR по chrF++ (быстрее, стабильнее)\"\"\"\n", " if len(candidates) <= 1:\n", " return candidates[0] if candidates else \"\"\n", "\n", " # Дедупликация\n", " seen = set()\n", " unique = []\n", " for c in candidates:\n", " if c not in seen:\n", " seen.add(c)\n", " unique.append(c)\n", " candidates = unique\n", "\n", " if len(candidates) == 1:\n", " return candidates[0]\n", "\n", " best_score = -1\n", " best_text = candidates[0]\n", "\n", " for i, hyp in enumerate(candidates):\n", " total = 0\n", " for j, ref in enumerate(candidates):\n", " if i != j:\n", " total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n", " avg = total / (len(candidates) - 1)\n", "\n", " if avg > best_score:\n", " best_score = avg\n", " best_text = hyp\n", "\n", " return best_text\n", "\n", "\n", "def mbr_decode_combined(candidates: List[str]) -> str:\n", " \"\"\"MBR по geometric mean(BLEU, chrF++) — как метрика соревнования\"\"\"\n", " if len(candidates) <= 1:\n", " return candidates[0] if candidates else \"\"\n", "\n", " seen = set()\n", " unique = []\n", " for c in candidates:\n", " if c not in seen:\n", " seen.add(c)\n", " unique.append(c)\n", " candidates = unique\n", "\n", " if len(candidates) == 1:\n", " return candidates[0]\n", "\n", " best_score = -1\n", " best_text = candidates[0]\n", "\n", " for i, hyp in enumerate(candidates):\n", " chrf_total = 0\n", " bleu_total = 0\n", "\n", " for j, ref in enumerate(candidates):\n", " if i != j:\n", " chrf_total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n", " bleu_total += sacrebleu.sentence_bleu(hyp, [ref], tokenize=\"intl\").score\n", "\n", " n = len(candidates) - 1\n", " avg_chrf = chrf_total / n\n", " avg_bleu = bleu_total / n\n", "\n", " if avg_bleu > 0 and avg_chrf > 0:\n", " combined = math.sqrt(avg_bleu * avg_chrf)\n", " else:\n", " combined = avg_chrf\n", "\n", " if combined > best_score:\n", " best_score = combined\n", " best_text = hyp\n", "\n", " return best_text\n", "\n", "\n", "\n", "MBR_STRATEGIES = {\n", " \"chrf\": mbr_decode_chrf,\n", " \"combined\": mbr_decode_combined,\n", "}\n", "\n", "\n", "def select_best_candidates(\n", " candidates_df: pd.DataFrame,\n", " strategy: str = \"chrf\",\n", ") -> pd.DataFrame:\n", " \"\"\"\n", " Применяет MBR decoding к DataFrame с кандидатами.\n", "\n", " Returns:\n", " DataFrame с колонками: text_idx, input_text, prediction\n", " \"\"\"\n", " decode_fn = MBR_STRATEGIES[strategy]\n", " candidate_cols = get_candidate_columns(candidates_df)\n", "\n", " print(f\"\\nMBR Decoding (strategy={strategy})\")\n", " print(f\" Candidates per example: {len(candidate_cols)}\")\n", " print(f\" Total examples: {len(candidates_df)}\")\n", "\n", " predictions = []\n", " for _, row in tqdm(candidates_df.iterrows(),\n", " total=len(candidates_df), desc=\"MBR Decoding\"):\n", " cands = get_candidates_for_row(row, candidate_cols)\n", " best = decode_fn(cands)\n", " predictions.append(best)\n", "\n", " result = pd.DataFrame({\n", " \"text_idx\": candidates_df[\"text_idx\"],\n", " \"input_text\": candidates_df[\"input_text\"],\n", " \"prediction\": predictions,\n", " })\n", "\n", " return result\n", "\n", "\n", "# ============================================================\n", "# Этап 4: Оценка (валидация)\n", "# ============================================================\n", "\n", "def deep_past_score(predictions: List[str], references: List[str]) -> Dict[str, float]:\n", " \"\"\"Метрика соревнования: geometric mean(BLEU, chrF++)\"\"\"\n", " bleu = sacrebleu.corpus_bleu(predictions, [references], tokenize=\"intl\").score\n", " chrf = sacrebleu.corpus_chrf(predictions, [references], word_order=2).score\n", " score = math.sqrt(bleu * chrf) if bleu > 0 and chrf > 0 else 0.0\n", " return {\"bleu\": bleu, \"chrf++\": chrf, \"score\": score}\n", "\n", "\n", "def evaluate_predictions(\n", " predictions_df: pd.DataFrame,\n", " references: List[str],\n", ") -> Dict[str, float]:\n", " \"\"\"Оценить предсказания\"\"\"\n", " preds = predictions_df[\"prediction\"].tolist()\n", " metrics = deep_past_score(preds, references)\n", "\n", " print(f\"\\n{'='*40}\")\n", " print(f\"BLEU: {metrics['bleu']:.4f}\")\n", " print(f\"chrF++: {metrics['chrf++']:.4f}\")\n", " print(f\"Score: {metrics['score']:.4f}\")\n", " print(f\"{'='*40}\")\n", "\n", " return metrics\n", "\n", "\n", "def evaluate_per_model(\n", " candidates_df: pd.DataFrame,\n", " references: List[str],\n", " model_configs: List[ModelConfig],\n", ") -> pd.DataFrame:\n", " \"\"\"Оценить top-1 от каждой модели отдельно (для сравнения)\"\"\"\n", " results = []\n", "\n", " for config in model_configs:\n", " col = f\"{config.name}_0\" # top-1 кандидат\n", " if col in candidates_df.columns:\n", " preds = candidates_df[col].tolist()\n", " metrics = deep_past_score(preds, references)\n", " metrics[\"model\"] = config.name\n", " results.append(metrics)\n", " print(f\" {config.name}: score={metrics['score']:.4f} \"\n", " f\"(BLEU={metrics['bleu']:.2f}, chrF++={metrics['chrf++']:.2f})\")\n", "\n", " return pd.DataFrame(results)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ftbyg4uoDJhu" }, "outputs": [], "source": [ "# ============================================================\n", "# MBR Decoding\n", "# ============================================================\n", "\n", "def _dedup(candidates: List[str]) -> List[str]:\n", " \"\"\"Дедупликация с сохранением порядка\"\"\"\n", " seen = set()\n", " unique = []\n", " for c in candidates:\n", " c = str(c).strip()\n", " if c and c not in seen:\n", " seen.add(c)\n", " unique.append(c)\n", " return unique\n", "\n", "\n", "# chrF++ only\n", "\n", "def mbr_decode_chrf(candidates: List[str]) -> str:\n", " \"\"\"MBR по chrF++ (быстрее, стабильнее)\"\"\"\n", " if len(candidates) <= 1:\n", " return candidates[0] if candidates else \"\"\n", "\n", " # Дедупликация\n", " seen = set()\n", " unique = []\n", " for c in candidates:\n", " if c not in seen:\n", " seen.add(c)\n", " unique.append(c)\n", " candidates = unique\n", "\n", " if len(candidates) == 1:\n", " return candidates[0]\n", "\n", " best_score = -1\n", " best_text = candidates[0]\n", "\n", " for i, hyp in enumerate(candidates):\n", " total = 0\n", " for j, ref in enumerate(candidates):\n", " if i != j:\n", " total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n", " avg = total / (len(candidates) - 1)\n", "\n", " if avg > best_score:\n", " best_score = avg\n", " best_text = hyp\n", "\n", " return best_text\n", "\n", "\n", "def mbr_decode_combined(candidates: List[str]) -> str:\n", " \"\"\"MBR по geometric mean(BLEU, chrF++) — как метрика соревнования\"\"\"\n", " if len(candidates) <= 1:\n", " return candidates[0] if candidates else \"\"\n", "\n", " seen = set()\n", " unique = []\n", " for c in candidates:\n", " if c not in seen:\n", " seen.add(c)\n", " unique.append(c)\n", " candidates = unique\n", "\n", " if len(candidates) == 1:\n", " return candidates[0]\n", "\n", " best_score = -1\n", " best_text = candidates[0]\n", "\n", " for i, hyp in enumerate(candidates):\n", " chrf_total = 0\n", " bleu_total = 0\n", "\n", " for j, ref in enumerate(candidates):\n", " if i != j:\n", " chrf_total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n", " bleu_total += sacrebleu.sentence_bleu(hyp, [ref], tokenize=\"intl\").score\n", "\n", " n = len(candidates) - 1\n", " avg_chrf = chrf_total / n\n", " avg_bleu = bleu_total / n\n", "\n", " if avg_bleu > 0 and avg_chrf > 0:\n", " combined = math.sqrt(avg_bleu * avg_chrf)\n", " else:\n", " combined = avg_chrf\n", "\n", " if combined > best_score:\n", " best_score = combined\n", " best_text = hyp\n", "\n", " return best_text\n", "\n", "\n", "\n", "# Baseline-style weighted MBR ----\n", "\n", "def _jaccard(a: str, b: str) -> float:\n", " \"\"\"Token-level Jaccard similarity (0-100)\"\"\"\n", " ta = set(a.lower().split())\n", " tb = set(b.lower().split())\n", " if not ta and not tb:\n", " return 100.0\n", " if not ta or not tb:\n", " return 0.0\n", " return 100.0 * len(ta & tb) / len(ta | tb)\n", "\n", "\n", "def _length_bonus(lengths: List[int], idx: int) -> float:\n", " \"\"\"Гауссов бонус: близкие к медиане длины получают больше\"\"\"\n", " if not lengths:\n", " return 100.0\n", " median = float(np.median(lengths))\n", " sigma = max(median * 0.4, 5.0)\n", " z = (lengths[idx] - median) / sigma\n", " return 100.0 * math.exp(-0.5 * z * z)\n", "\n", "\n", "def mbr_decode_baseline(\n", " candidates: List[str],\n", " w_chrf: float = 0.55,\n", " w_bleu: float = 0.25,\n", " w_jaccard: float = 0.20,\n", " w_length: float = 0.10,\n", " pool_cap: int = 32,\n", ") -> str:\n", " \"\"\"\n", " Baseline MBR: взвешенная комбинация chrF++ + BLEU + Jaccard + length bonus.\n", " \"\"\"\n", " candidates = _dedup(candidates)\n", " if pool_cap:\n", " candidates = candidates[:pool_cap]\n", "\n", " n = len(candidates)\n", " if n == 0:\n", " return \"\"\n", " if n == 1:\n", " return candidates[0]\n", "\n", " pw_total = max(w_chrf + w_bleu + w_jaccard, 1e-9)\n", " lengths = [len(c.split()) for c in candidates]\n", "\n", " best_score = -1\n", " best_text = candidates[0]\n", "\n", " for i in range(n):\n", " pairwise_sum = 0\n", " for j in range(n):\n", " if j != i:\n", " chrf = sacrebleu.sentence_chrf(candidates[i], [candidates[j]], word_order=2).score\n", " bleu = sacrebleu.sentence_bleu(candidates[i], [candidates[j]], tokenize=\"intl\").score\n", " jacc = _jaccard(candidates[i], candidates[j])\n", "\n", " pairwise_sum += (w_chrf * chrf + w_bleu * bleu + w_jaccard * jacc) / pw_total\n", "\n", " avg_pairwise = pairwise_sum / (n - 1)\n", " lb = _length_bonus(lengths, i)\n", " total = avg_pairwise + w_length * lb\n", "\n", " if total > best_score:\n", " best_score = total\n", " best_text = candidates[i]\n", "\n", " return best_text\n", "\n", "\n", "\n", "\n", "MBR_STRATEGIES = {\n", " \"chrf\": mbr_decode_chrf,\n", " \"combined\": mbr_decode_combined,\n", " \"baseline_chrf_heavy\": lambda c: mbr_decode_baseline(c, w_chrf=0.80, w_bleu=0.10, w_jaccard=0.10, w_length=0.05),\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "5883d515c08a460eb9997946d7b4f382", "6cfd690ef40a46708359d7ba3503c4a3", "1f9e1517c58e437eab05851cd3297c2e", "8c0a59748d4b4646a97d44d8c34c2a48", "83bb6f9715964d218fd5fa3ba87919a9", "08074d039bc04e72a4587a60bbaade39", "2e89a4d620d14611824563b00f4ee97f", "d689d564b506463897afb007ad733af4", "1ff4129ba9de4f59a502a77e027672df", "1aa7ca7f8647463d94a975bb042368d0", "7e89e4d3fea244f387d0e999c11f6a1c" ] }, "id": "ScotFFXJBfuB", "outputId": "18e85644-52eb-42b0-fe30-cc6698612f60" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "[mix_v2] Loading model from /content/drive/MyDrive/Deep Past Challenge 26/models/LARGE_MIXES/LARGE_MIXES_WITH_OLD_NEW_DATA_v2\n", " beams=6, return=3, lp=1.7, batch=5\n", "============================================================\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[mix_v2] Generating: 100%|██████████| 40/40 [01:48<00:00, 2.72s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[mix_v2] Done. Generated 198 × 3 candidates. VRAM free: 102.0 GB\n", "\n", "============================================================\n", "[drop_v1] Loading model from /content/drive/MyDrive/Deep Past Challenge 26/models/LARGE_MIXES/LARGE_MIXES_TRAINED_WITH_TEST_DROP_v1\n", " beams=6, return=3, lp=1.7, batch=5\n", "============================================================\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[drop_v1] Generating: 100%|██████████| 40/40 [01:46<00:00, 2.66s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[drop_v1] Done. Generated 198 × 3 candidates. VRAM free: 102.0 GB\n", "\n", "============================================================\n", "[mt5_large_drop] Loading model from /content/drive/MyDrive/Deep Past Challenge 26/models/aduah_MT_LARGE_from_zero_v1_continue_drop/checkpoint-1455\n", " beams=6, return=3, lp=1.7, batch=5\n", "============================================================\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[mt5_large_drop] Generating: 100%|██████████| 40/40 [00:49<00:00, 1.23s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[mt5_large_drop] Done. Generated 198 × 3 candidates. VRAM free: 102.0 GB\n", "\n", "============================================================\n", "[mt5_xl_bf16] Loading model from /content/drive/MyDrive/Deep Past Challenge 26/models/XL_runpod_checkpoint-8190_bf16\n", " beams=5, return=1, lp=1.7, batch=5\n", "============================================================\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5883d515c08a460eb9997946d7b4f382", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/2 [00:00