{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":14793727,"datasetId":9458259,"databundleVersionId":15648092},{"sourceType":"modelInstanceVersion","sourceId":776508,"databundleVersionId":15960595,"modelInstanceId":592811,"modelId":605101,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":785188,"databundleVersionId":16063480,"modelInstanceId":599133,"modelId":611388,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":779991,"databundleVersionId":15996471,"modelInstanceId":595149,"modelId":607402,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":777826,"databundleVersionId":15974476,"modelInstanceId":593631,"modelId":605899,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":779068,"databundleVersionId":15987045,"modelInstanceId":594497,"modelId":606762,"isSourceIdPinned":false}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport math\nimport pandas as pd\nimport torch\nimport gc\nfrom collections import Counter\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nfrom peft import PeftModel","metadata":{"_uuid":"d5fea252-0577-49c3-bacd-e050f99f77e3","_cell_guid":"96728166-ef48-475e-8eb7-382a71569fb5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BEST_BASE_MODEL = \"/kaggle/input/models/dmitrysokolevskiy/dpc-byt5-large-512-ai-translation-rerun-v3/transformers/default/1\"\n\n# Список всех LoRA, которые вы хотите смешать\nLORA_CONFIGS = [\n    {\n        \"path\": \"/kaggle/input/models/dmitrysokolevskiy/u2-lora-dpc-byt5-large-v5.1_fullt/transformers/default/1\", #36.8 (Основная база)\n        \"w\": 0.45,\n        \"drop_rate\": 0.0 # Оставляем все веса как есть\n    },\n    {\n        \"path\": \"/kaggle/input/models/dmitrysokolevskiy/u2-lora-dpc-byt5-large-v5-2-fullt-18e/transformers/default/1\", #36.4 (Сильный помощник)\n        \"w\": 0.20,\n        \"drop_rate\": 0.1 # Слегка прореживаем, чтобы уменьшить шум\n    },\n    {\n        \"path\": \"/kaggle/input/models/dmitrysokolevskiy/u2-lora-dpc-byt5-large-v5-3-fullt-13e/transformers/default/1\", #35.7 (Спец по именам)\n        \"w\": 0.10,\n        \"drop_rate\": 0.7 # DARE: выкидываем 70% весов, чтобы не сломать грамматику основных моделей\n    },\n    {\n        \"path\": \"/kaggle/input/models/dmitrysokolevskiy/u2-lora-dpc-byt5-large-all-from-start-ds-s-v6/transformers/default/1\", #\n        \"w\": 0.25,\n        \"drop_rate\": 0.5\n    }\n]\n\n\ntotal_weight = sum(c['w'] for c in LORA_CONFIGS)\nprint(f\"Total weight sum: {total_weight}\")","metadata":{"_uuid":"c909461c-45e6-4e64-8539-a5110a0ab933","_cell_guid":"583ebf6b-9608-4aa5-b735-94dc155fc318","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nBATCH_SIZE = 8\n\nMAX_LENGTH = 512\n\nGEN_PARAMS = {\n    \"num_beams\": 5,\n    \"max_new_tokens\": 512,\n    \"length_penalty\": 0.9,    # Можно поиграть (0.6 - 1.2)\n    \"early_stopping\": True,    \n}\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"_uuid":"ff0334ae-ca44-4e24-a722-07feae282c26","_cell_guid":"2e193e76-a029-4a1a-aad2-34aae1a47ac2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nPREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [PREFIX + str(i) for i in self.texts]\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=MAX_LENGTH, \n            padding=\"max_length\",\n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }","metadata":{"_uuid":"16f10608-7e02-462f-9155-53f298c1107a","_cell_guid":"648c0507-050d-4fe4-962c-4078e8edeb13","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_dare(delta, drop_rate):\n    \"\"\"Применяет DARE: зануляет случайные веса и масштабирует остальные.\"\"\"\n    if drop_rate <= 0.0:\n        return delta\n    # Создаем маску: 1 для весов, которые оставляем, 0 для тех, что выкидываем\n    mask = (torch.rand_like(delta) > drop_rate).to(delta.dtype)\n    # Оставляем только выбранные веса и масштабируем их\n    return delta * mask / (1.0 - drop_rate)\n\ndef create_lora_soup(base_model_path, lora_configs, device):\n    print(\"Loading Base Model State Dict for Task Vectors...\")\n    base_model = AutoModelForSeq2SeqLM.from_pretrained(\n        base_model_path, \n        torch_dtype=torch.float32, \n        device_map=\"cpu\"\n    )\n    base_state_dict = {k: v.clone() for k, v in base_model.state_dict().items()}\n    \n    # Словарь для хранения нашей итоговой \"дельты\" (Task Vector)\n    soup_delta_dict = {k: torch.zeros_like(v) for k, v in base_state_dict.items()}\n    \n    del base_model\n    gc.collect()\n    \n    for i, conf in enumerate(tqdm(lora_configs, desc=\"Mixing LoRA Models (DARE)\")):\n        path = conf[\"path\"]\n        weight = conf[\"w\"]\n        drop_rate = conf.get(\"drop_rate\", 0.0)\n        \n        print(f\"\\nProcessing LoRA #{i+1} | Weight: {weight} | Drop Rate: {drop_rate}\")\n        \n        # Снова загружаем базовую модель для слияния с конкретной LoRA\n        temp_base = AutoModelForSeq2SeqLM.from_pretrained(\n            base_model_path, \n            torch_dtype=torch.float32, \n            device_map=\"cpu\"\n        )\n        peft_model = PeftModel.from_pretrained(temp_base, path, device_map=\"cpu\")\n        merged_model = peft_model.merge_and_unload()\n        merged_model.eval()\n        \n        current_state_dict = merged_model.state_dict()\n        \n        print(\"  Calculating Delta and applying DARE...\")\n        for key in current_state_dict:\n            # 1. Вычисляем дельту (Task Vector): насколько LoRA изменила базовую модель\n            delta = current_state_dict[key].float() - base_state_dict[key].float()\n            \n            # 2. Применяем DARE (прореживание)\n            dare_delta = apply_dare(delta, drop_rate)\n            \n            # 3. Взвешиваем и добавляем в общий котел дельт\n            soup_delta_dict[key] += dare_delta * weight\n            \n        del temp_base, peft_model, merged_model, current_state_dict\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    print(\"\\nLoading Final Architecture for Soup...\")\n    final_model = AutoModelForSeq2SeqLM.from_pretrained(\n        base_model_path, \n        torch_dtype=torch.float32, \n        device_map=\"cpu\"\n    )\n    \n    print(\"Applying Soup Delta to Base Model...\")\n    final_state_dict = {}\n    for key in base_state_dict:\n        # Итоговый вес = Базовый вес + (Сумма всех DARE дельт)\n        final_state_dict[key] = base_state_dict[key] + soup_delta_dict[key]\n        \n    final_model.load_state_dict(final_state_dict)\n    \n    print(f\"Moving Final Soup to {device}...\")\n    final_model = final_model.to(device)\n    \n    print(\"DARE LoRA Soup is served! 🍲\")\n    return final_model","metadata":{"_uuid":"969cd368-2e97-4059-9a4f-460f34a75e3e","_cell_guid":"c5d8c02c-3420-43e9-baf8-7b7e6ffe41fa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(TEST_DATA_PATH)\n\nmodel = create_lora_soup(BEST_BASE_MODEL, LORA_CONFIGS, DEVICE)\n\ntokenizer = AutoTokenizer.from_pretrained(BEST_BASE_MODEL)\n\ndataset = InferenceDataset(df, tokenizer)\nloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False)\n\npredictions = []\n\nprint(\"Running Inference...\")\nwith torch.no_grad():\n    for batch in tqdm(loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n        \n        outputs = model.generate(\n            input_ids=input_ids, \n            attention_mask=attention_mask, \n            **GEN_PARAMS\n        )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        cleaned = [t.strip()  for t in decoded]\n        predictions.extend(cleaned)\n\nsubmission = pd.DataFrame({\"id\": df[\"id\"], \"translation\": predictions})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Saved submission.csv\")","metadata":{"_uuid":"801576c2-1622-4767-960b-f531efca25e8","_cell_guid":"6712730a-07f7-4de4-a88c-b41e9c20a2c0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"91ba6436-62dd-4f20-ad37-c702758a8209","_cell_guid":"6cf22e9b-c9d0-4411-8844-0fee124dd898","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}