{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#импорт библиотек \n\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport time\n\n#для работы gpu в Кaggle\n#!pip uninstall torch torchvision torchaudio -y\n#!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"Все библиотеки импортированы\")\n\n\n\n# загрузка данных (KAGGLE):\ndata_dir = '/kaggle/input/competitions/butterflies-classification'\n\n\ntrain_dir = os.path.join(data_dir, 'train_butterflies', 'train_split')\ntest_dir = os.path.join(data_dir, 'test_butterflies', 'valid')\n\n\nprint(f\"data_dir: {data_dir}\")\nprint(f\"train_dir: {train_dir}\")\nprint(f\"test_dir: {test_dir}\")\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:04:54.499644Z","iopub.execute_input":"2026-07-27T21:04:54.50034Z","iopub.status.idle":"2026-07-27T21:04:59.097525Z","shell.execute_reply.started":"2026-07-27T21:04:54.500308Z","shell.execute_reply":"2026-07-27T21:04:59.09679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#1. Предварительное исследование\n\n#1.1 Анализ баланса классов\n\nclass_counts = {}\nclass_names = sorted([d for d in os.listdir(train_dir) \n                     if os.path.isdir(os.path.join(train_dir, d))])\n\nfor class_name in class_names:\n    class_path = os.path.join(train_dir, class_name)\n    num_images = len(os.listdir(class_path))\n    class_counts[class_name] = num_images\n\n# Создаем DataFrame \ndf_classes = pd.DataFrame({\n    'Class': list(class_counts.keys()),\n    'Count': list(class_counts.values())\n}).sort_values('Count', ascending=False)\n\nprint(f\"\\n Статистика по классам:\")\nprint(f\"- Всего классов: {len(class_counts)}\")\nprint(f\"- Всего изображений: {sum(class_counts.values())}\")\nprint(f\"- Минимум: {df_classes['Count'].min()}\")\nprint(f\"- Максимум: {df_classes['Count'].max()}\")\nprint(f\"- Среднее: {df_classes['Count'].mean():.1f}\")\nprint(f\"- Медиана: {df_classes['Count'].median():.1f}\")\nprint(f\"- Разница между max и min: {df_classes['Count'].max() - df_classes['Count'].min()}\")\n\n# Оценка баланса\nmin_count = df_classes['Count'].min()\nmax_count = df_classes['Count'].max()\nratio = max_count / min_count\nif ratio < 1.5:\n    print(f\"Классы сбалансированы (соотношение max/min = {ratio:.2f})\")\nelif ratio < 2.5:\n    print(f\"Небольшой дисбаланс классов (соотношение max/min = {ratio:.2f})\")\nelse:\n    print(f\"Сильный дисбаланс классов (соотношение max/min = {ratio:.2f})\")\n    print(f\"Рекомендуется использовать веса классов или аугментацию\")\n\n# Визуализация распределения классов\nplt.figure(figsize=(15, 6))\nsns.barplot(data=df_classes, x='Class', y='Count', palette='viridis')\nplt.xticks(rotation=90)\nplt.title('Распределение изображений по классам', fontsize=14)\nplt.xlabel('Класс')\nplt.ylabel('Количество изображений')\nplt.axhline(y=df_classes['Count'].mean(), color='red', linestyle='--', \n            label=f'Среднее: {df_classes[\"Count\"].mean():.1f}')\nplt.tight_layout()\nplt.show()\n\n# 1.2 Проверка размеров изображений\n\nprint(\"\\n Проверка размеров изображений:\")\nsizes = {}\nsample_classes = np.random.choice(class_names, min(5, len(class_names)), replace=False)\n\nfor class_name in sample_classes:\n    class_path = os.path.join(train_dir, class_name)\n    img_files = os.listdir(class_path)[:10]  \n    for img_name in img_files:\n        img_path = os.path.join(class_path, img_name)\n        try:\n            with Image.open(img_path) as img:\n                sizes[img_path] = img.size\n        except Exception as e:\n            print(f\"Ошибка при открытии {img_path}: {e}\")\n\nunique_sizes = set(sizes.values())\nprint(f\"- Уникальные размеры: {unique_sizes}\")\nif len(unique_sizes) == 1:\n    print(f\"Все изображения имеют размер {unique_sizes.pop()}\")\nelse:\n    print(f\"Обнаружены разные размеры\")\n\n#1.3 Визуализация примеров изображений\n\ndef show_examples(num_examples=5):\n    fig, axes = plt.subplots(1, num_examples, figsize=(15, 4))\n    selected_classes = np.random.choice(class_names, num_examples, replace=False)\n    \n    for i, class_name in enumerate(selected_classes):\n        class_path = os.path.join(train_dir, class_name)\n        img_files = [f for f in os.listdir(class_path) \n                    if f.endswith(('.jpg', '.jpeg', '.png'))]\n        if img_files:\n            img_path = os.path.join(class_path, np.random.choice(img_files))\n            img = Image.open(img_path)\n            axes[i].imshow(img)\n            axes[i].set_title(f'Класс: {class_name[:20]}\\nРазмер: {img.size}')\n            axes[i].axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\nprint(\"\\n Примеры изображений из разных классов:\")\nshow_examples(5)\n\n# Анализ особенностей\nprint(\"\\n ОСОБЕННОСТИ ДАТАСЕТА:\")\nprint(\"1. Изображения бабочек на естественном фоне\")\nprint(\"2. Разные ракурсы и освещение\")\nprint(\"3. Некоторые изображения могут быть размытыми\")\nprint(\"4. Фон может содержать другие объекты (цветы, листья)\")\n\n#1.4 Проверка на битые изображения\n\nprint(\"\\n Проверка целостности изображений:\")\nbroken_images = []\nfor class_name in tqdm(class_names, desc=\"Проверка\"):\n    class_path = os.path.join(train_dir, class_name)\n    for img_name in os.listdir(class_path):\n        img_path = os.path.join(class_path, img_name)\n        try:\n            with Image.open(img_path) as img:\n                img.verify()\n        except Exception:\n            broken_images.append(img_path)\n\nprint(f\"- Битых изображений: {len(broken_images)}\")\nif len(broken_images) == 0:\n    print(\"Все изображения целые\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:05:19.066227Z","iopub.execute_input":"2026-07-27T21:05:19.067286Z","iopub.status.idle":"2026-07-27T21:05:27.909627Z","shell.execute_reply.started":"2026-07-27T21:05:19.067254Z","shell.execute_reply":"2026-07-27T21:05:27.908989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#2. Подготовка для обучения\n\n#2.1 Создание датасета\n\nclass MyDataset(Dataset):\n    \n    def __init__(self, data_dir, transform=None, is_train=True):\n        self.data_dir = data_dir\n        self.transform = transform\n        self.is_train = is_train\n        \n        # Списки для хранения путей и меток\n        self.images = []      # Список путей к изображениям\n        self.labels = []      # Список меток (числовых индексов)\n        self.class_to_idx = {}  # Словарь соответствия класса индексу\n        self.classes = []     # Список названий классов\n        \n        if is_train:\n                      \n            all_items = os.listdir(data_dir)\n            class_folders = [\n                d for d in all_items \n                if os.path.isdir(os.path.join(data_dir, d)) \n                and d.startswith('class_')]\n\n            self.classes = sorted(class_folders, key=lambda x: int(x.split('_')[1])) \n            self.class_to_idx = {\n                cls_name: idx for idx, cls_name in enumerate(self.classes)}\n            \n            print(f\"Найдено классов: {len(self.classes)}\")\n            \n            for class_name in self.classes:\n                class_path = os.path.join(data_dir, class_name)\n                for img_name in os.listdir(class_path):\n                    if img_name.lower().endswith(('.jpg', '.jpeg', '.png')):\n                        self.images.append(os.path.join(class_path, img_name))\n                        self.labels.append(self.class_to_idx[class_name])\n            \n            print(f\"Загружено изображений: {len(self.images)}\")\n            \n        else:\n            \n            all_files = sorted([f for f in os.listdir(data_dir) \n                               if f.lower().endswith(('.jpg', '.jpeg', '.png'))])\n            \n            self.images = [\n                os.path.join(data_dir, img_name)\n                for img_name in all_files\n                if img_name.lower().endswith(('.jpg', '.jpeg', '.png'))]\n            \n            self.labels = [-1] * len(self.images)\n            \n            print(f\"Загружено тестовых изображений: {len(self.images)}\")\n    \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        img_path = self.images[idx]\n        \n        try:\n            image = Image.open(img_path).convert('RGB')\n            if self.transform:\n                image = self.transform(image)\n            label = self.labels[idx]\n            return image, label, img_path\n            \n        except Exception as e:\n            print(f\"Ошибка загрузки {img_path}: {e}\")\n            dummy_image = torch.zeros(3, 224, 224)  # Пустое изображение\n            return dummy_image, -1, img_path\n\n#2.2 Трансформации\n\ntrain_transform = transforms.Compose([\n    transforms.Resize((256, 256)),  \n    transforms.RandomCrop(224),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(\n        brightness=0.2,  \n        contrast=0.2,   \n        saturation=0.2,  \n        hue=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],  \n        std=[0.229, 0.224, 0.225])\n])\n\nvalid_transform = transforms.Compose([\n    transforms.Resize((224, 224)),  \n    transforms.ToTensor(),          \n    transforms.Normalize(           \n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225])\n])\n\nprint(\"Трансформации для тренировки (с аугментацией) проведены\")\nprint(\"Трансформации для валидации (без аугментации) проведены\")\n\n#2.3 Создание датасетов\n\n#тренировочный\nfull_train_dataset = MyDataset(\n    data_dir=train_dir,\n    transform=train_transform, \n    is_train=True\n)\n\n#тестовый\ntest_dataset = MyDataset(\n    data_dir=test_dir,\n    transform=valid_transform,  \n    is_train=False\n)\n\nprint(f\"\\n Размеры датасетов:\")\nprint(f\"- Тренировочный: {len(full_train_dataset)}\")\nprint(f\"- Тестовый: {len(test_dataset)}\")\n\n#2.4 Разделение на выборки\n\ntrain_ratio = 0.8   # 80% на обучение\nvalid_ratio = 0.2   # 20% на валидацию\n\ntrain_size = int(train_ratio * len(full_train_dataset))\nvalid_size = len(full_train_dataset) - train_size\n\nprint(f\"- Соотношение: {train_ratio*100:.0f}% / {valid_ratio*100:.0f}%\")\nprint(f\"- Тренировочных: {train_size}\")\nprint(f\"- Валидационных: {valid_size}\")\n\ntrain_dataset, valid_dataset = random_split(\n    full_train_dataset,\n    [train_size, valid_size],\n    generator=torch.Generator().manual_seed(42))\n\nprint(\"Данные разделены\")\n\n#2.5 DataLoader\n\nbatch_size = 32\nnum_workers = 2\npin_memory = True\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=batch_size,\n    shuffle=True,      \n    num_workers=num_workers,\n    pin_memory=pin_memory,\n    drop_last=False)\n\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=batch_size,\n    shuffle=False,     \n    num_workers=num_workers,\n    pin_memory=pin_memory,\n    drop_last=False)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=batch_size,\n    shuffle=False,     \n    num_workers=num_workers,\n    pin_memory=pin_memory,\n    drop_last=False)\n\nprint(f\"- Размер батча: {batch_size}\")\nprint(f\"- Количество батчей в train: {len(train_loader)}\")\nprint(f\"- Количество батчей в valid: {len(valid_loader)}\")\nprint(f\"- Количество батчей в test: {len(test_loader)}\")\n\n#2.6 Проверка\n\n#Один батч из тренировочных данных\ndef show_batch(loader, num_images=8, title=\"Примеры изображений\"):\n    images, labels, paths = next(iter(loader))\n    images = images[:num_images]\n    labels = labels[:num_images]\n    cols = 4\n    rows = (num_images + cols - 1) // cols\n    fig, axes = plt.subplots(rows, cols, figsize=(12, 3*rows))\n    axes = axes.flatten()\n    mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n    std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n    \n    for i in range(num_images):\n        img = images[i]\n        img = img * std + mean\n        img = torch.clamp(img, 0, 1)\n        img = img.permute(1, 2, 0).numpy()\n        axes[i].imshow(img)\n        axes[i].set_title(f'Метка: {labels[i].item()}')\n        axes[i].axis('off')\n\n    for i in range(num_images, len(axes)):\n        axes[i].axis('off')\n    \n    plt.suptitle(title, fontsize=14)\n    plt.tight_layout()\n    plt.show()\n\nshow_batch(train_loader, num_images=8, title=\"Примеры из тренировочной выборки\")\n\nprint(\"Данные готовы к обучению\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:05:41.897704Z","iopub.execute_input":"2026-07-27T21:05:41.898521Z","iopub.status.idle":"2026-07-27T21:05:43.373297Z","shell.execute_reply.started":"2026-07-27T21:05:41.898482Z","shell.execute_reply":"2026-07-27T21:05:43.371744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#3. Реализация модели\n \n#3.1 Класс модели\n\nclass RSNAModel(nn.Module):\n  \n    def __init__(self, num_classes=50, dropout_rate=0.3, pretrained=True):\n        super(RSNAModel, self).__init__()\n        self.backbone = models.resnet50(\n            weights=models.ResNet50_Weights.IMAGENET1K_V1 if pretrained else None) #предобученная\n        \n        for param in self.backbone.parameters():\n            param.requires_grad = False\n        \n        for param in self.backbone.layer4.parameters():\n            param.requires_grad = True\n        \n        num_features = self.backbone.fc.in_features  # 2048\n        \n        #Новый классификатор\n        self.backbone.fc = nn.Sequential(\n            nn.Dropout(dropout_rate),     \n            nn.Linear(num_features, 512), \n            nn.ReLU(),                     \n            nn.Dropout(dropout_rate),     \n            nn.Linear(512, num_classes))\n        \n        print(f\"Модель создана:\")\n        print(f\"- Архитектура: ResNet50\")\n        print(f\"- Классов: {num_classes}\")\n        print(f\"- Dropout: {dropout_rate}\")\n        print(f\"- Предобучена: {pretrained}\")\n    \n    def forward(self, x):\n        return self.backbone(x)\n\n\n#3.2 Обучение 1 эпохи\n\ndef train_one_epoch(model, train_dataloader, optimizer, loss_fn, epoch, device, \n                    log_wandb=True, verbose=False):\n   \n    model.train()\n    \n    running_loss = 0.0          # Суммарная потеря за эпоху\n    correct = 0                 # Количество правильных ответов\n    total = 0                   # Всего примеров\n    batch_count = 0             # Счетчик батчей\n    \n    #Визуализация прогресса\n    pbar = tqdm(train_dataloader, desc=f\"Epoch {epoch+1} [Train]\", \n                disable=not verbose)\n    \n    for batch_idx, (images, labels, paths) in enumerate(pbar):\n        images = images.to(device)\n        labels = labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        #статистика\n        running_loss += loss.item()  \n        _, predicted = outputs.max(1)  #берем класс с максимальной вероятностью\n        total += labels.size(0)        \n        correct += predicted.eq(labels).sum().item() \n        \n        batch_count += 1\n        \n        #обновление прогресса(визуализация) - каждые 10 батчей\n        if verbose and batch_idx % 10 == 0:\n            avg_loss = running_loss / batch_count\n            avg_acc = 100. * correct / total\n            pbar.set_postfix({\n                'Loss': f'{avg_loss:.4f}',\n                'Acc': f'{avg_acc:.2f}%'})\n    \n    #итоговые метрики за эпоху\n    epoch_loss = running_loss / len(train_dataloader)\n    epoch_acc = 100. * correct / total\n    \n    return epoch_loss, epoch_acc\n\n\n#3.3 Валидация 1 эпохи\n@torch.no_grad()\ndef valid_one_epoch(model, valid_dataloader, loss_fn, epoch, device, \n                    log_wandb=True, verbose=False):\n    model.eval()\n    \n    running_loss = 0.0\n    correct = 0\n    total = 0\n    batch_count = 0\n    \n    #прогресс\n    pbar = tqdm(valid_dataloader, desc=f\"Epoch {epoch+1} [Valid]\", \n                disable=not verbose)\n    \n    for batch_idx, (images, labels, paths) in enumerate(pbar):\n        images = images.to(device)\n        labels = labels.to(device)\n        outputs = model(images)\n        loss = loss_fn(outputs, labels)\n        #статистика\n        running_loss += loss.item()\n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n        \n        batch_count += 1\n        \n        #обновление прогресса\n        if verbose and batch_idx % 10 == 0:\n            avg_loss = running_loss / batch_count\n            avg_acc = 100. * correct / total\n            pbar.set_postfix({\n                'Loss': f'{avg_loss:.4f}',\n                'Acc': f'{avg_acc:.2f}%'})\n    \n    #итоговые метрики\n    epoch_loss = running_loss / len(valid_dataloader)\n    epoch_acc = 100. * correct / total\n    \n    return epoch_loss, epoch_acc\n\n\n#3.4 Предсказания\n\n@torch.no_grad()\ndef predict(model, test_dataloader, device):\n    model.eval()\n    predictions = []\n    img_paths = []\n    \n    for images, _, paths in tqdm(test_dataloader, desc=\"Predicting\"):\n        images = images.to(device)\n        outputs = model(images)\n        _, preds = outputs.max(1)\n        predictions.extend(preds.cpu().numpy())\n        img_paths.extend(paths)\n    \n    return predictions, img_paths\n\n\n#3.5 Создание модели\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #если можно использовать gpu\nprint(f\"Устройство: {device}\")\n\nnum_classes = len(full_train_dataset.classes)  \nmodel = RSNAModel(num_classes=num_classes, dropout_rate=0.3)\nmodel = model.to(device)\n\n#параметры\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nprint(f\"\\n Параметры модели:\")\nprint(f\"- Всего: {total_params:,}\")\nprint(f\"- Обучаемых: {trainable_params:,} ({(trainable_params/total_params)*100:.1f}%)\")\n\n#оптимизация\nlearning_rate = 1e-3  \noptimizer = optim.Adam([\n    {'params': model.backbone.layer4.parameters(), 'lr': learning_rate * 0.1},\n    {'params': model.backbone.fc.parameters(), 'lr': learning_rate}\n], lr=learning_rate, weight_decay=1e-4)  #weight_decay - L2 регуляризация\n\nprint(f\"\\n Параметры оптимизатора:\")\nprint(f\"- Optimizer: Adam\")\nprint(f\"- Learning rate: {learning_rate}\")\nprint(f\"- Weight decay: 1e-4\")\n\n#функция потерь\ncriterion = nn.CrossEntropyLoss()\nprint(f\"- Loss function: CrossEntropyLoss\")\n\n#планировщик скорости обучения\nscheduler = optim.lr_scheduler.CosineAnnealingLR(\n    optimizer, \n    T_max=15,  # Количество эпох\n    eta_min=1e-6  # Минимальный learning rate\n)\nprint(f\"- Scheduler: CosineAnnealingLR (T_max=15)\")\n\nprint(\"\\n Модель готова к обучению\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:05:50.230554Z","iopub.execute_input":"2026-07-27T21:05:50.231028Z","iopub.status.idle":"2026-07-27T21:05:50.794101Z","shell.execute_reply.started":"2026-07-27T21:05:50.230992Z","shell.execute_reply":"2026-07-27T21:05:50.793166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#4. Обучение модели\n\nnum_epochs = 15   \n\n#списки для хранения истории обучения\ntrain_losses = []\ntrain_accs = []\nvalid_losses = []\nvalid_accs = []\n\n#переменная для лучшей модели\nbest_valid_acc = 0.0\nbest_model_state = None\n\n#папка для сохранения модели\nos.makedirs('./models', exist_ok=True)\n\nprint(\"\\n Начало обучения\")\nstart_time = time.time()\n\n#4.1 Обучение\nfor epoch in range(num_epochs):\n    print(f\"\\n Эпоха {epoch+1}/{num_epochs}\")\n    \n    train_loss, train_acc = train_one_epoch(\n        model=model,\n        train_dataloader=train_loader,\n        optimizer=optimizer,\n        loss_fn=criterion,\n        epoch=epoch,\n        device=device,\n        log_wandb=False,\n        verbose=True)  #прогресс\n    \n    valid_loss, valid_acc = valid_one_epoch(\n        model=model,\n        valid_dataloader=valid_loader,\n        loss_fn=criterion,\n        epoch=epoch,\n        device=device,\n        log_wandb=False,\n        verbose=True)  #прогресс\n    \n    scheduler.step()\n    current_lr = scheduler.get_last_lr()[0]\n    \n    train_losses.append(train_loss)\n    train_accs.append(train_acc)\n    valid_losses.append(valid_loss)\n    valid_accs.append(valid_acc)\n    \n    #промежуточные результаты\n    print(f\"\\n Результаты эпохи {epoch+1}:\")\n    print(f\"- Train Loss: {train_loss:.4f}\")\n    print(f\"- Train Accuracy: {train_acc:.2f}%\")\n    print(f\"- Valid Loss: {valid_loss:.4f}\")\n    print(f\"- Valid Accuracy: {valid_acc:.2f}%\")\n    print(f\"- Learning Rate: {current_lr:.6f}\")\n    \n    if valid_acc > best_valid_acc:\n        best_valid_acc = valid_acc\n        best_model_state = model.state_dict().copy()\n        torch.save(best_model_state, './models/best_model.pth')\n        print(f\"Новая лучшая модель. Accuracy: {best_valid_acc:.2f}%\")\n    \n#3.2 Итоги\nprint(\"\\n Итоги\")\n\ntotal_time = time.time() - start_time\nprint(f\"\\n Общее время обучения: {total_time:.2f} секунд ({total_time/60:.2f} минут)\")\nprint(f\"\\n Лучшая точность на валидации: {best_valid_acc:.2f}%\")\n\n#загрузка лучшей модели\nmodel.load_state_dict(best_model_state)\nprint(\"Загружена лучшая модель\")\n\n\n#4.3 Предсказания\n\ntest_predictions, test_paths = predict(model, test_loader, device)\nprint(f\"Сделано предсказаний: {len(test_predictions)}\")\n\n#4.4 Submission \n\ndf = pd.DataFrame({\n    'path': test_paths,\n    'label': test_predictions})\n\n#извлекаем номер из названия\ndf['index'] = df['path'].apply(lambda x: int(os.path.basename(x).split('.')[0]))\n\nprint(df.head())\n\ndf = df.sort_values('index').reset_index(drop=True)\n\nsubmission = pd.DataFrame({\n    'index': df['index'],\n    'label': df['label']})\n\n#csv\nsubmission.to_csv('submission.csv', index=False)\nprint(f\"Файл submission.csv создан!\")\nprint(f\"Размер: {len(submission)} строк\")\nprint(f\"Первые 5 строк:\")\nprint(submission.head())\n\n\nprint(\"\\n Обучение завершено.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:05:56.614925Z","iopub.execute_input":"2026-07-27T21:05:56.615885Z","iopub.status.idle":"2026-07-27T21:12:36.337825Z","shell.execute_reply.started":"2026-07-27T21:05:56.615851Z","shell.execute_reply":"2026-07-27T21:12:36.33677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#5. Экспериментальная часть\n\n#5.1 Создание моделей\n\nclass ModelFactory:\n    \n    @staticmethod\n    def create_model(model_name, num_classes=50, dropout_rate=0.3):\n        if model_name == 'resnet50':\n            return ModelFactory._create_resnet50(num_classes, dropout_rate)\n        elif model_name == 'efficientnet':\n            return ModelFactory._create_efficientnet(num_classes, dropout_rate)\n        elif model_name == 'densenet':\n            return ModelFactory._create_densenet(num_classes, dropout_rate)\n        else:\n            raise ValueError(f\"Неизвестная модель: {model_name}\")\n    \n    @staticmethod\n    def _create_resnet50(num_classes, dropout_rate):\n        model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)\n       \n        for param in model.parameters():\n            param.requires_grad = False\n        \n        for param in model.layer4.parameters():\n            param.requires_grad = True\n        \n        num_features = model.fc.in_features\n        model.fc = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(),\n            nn.Dropout(dropout_rate),\n            nn.Linear(512, num_classes))\n        \n        return model\n    \n    @staticmethod\n    def _create_efficientnet(num_classes, dropout_rate):\n        \n        model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1)\n        \n        for param in model.parameters():\n            param.requires_grad = False\n        \n        for param in model.features[-3:].parameters():\n            param.requires_grad = True\n        \n        num_features = model.classifier[1].in_features\n        model.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(),\n            nn.Dropout(dropout_rate),\n            nn.Linear(512, num_classes))\n        \n        return model\n    \n    @staticmethod\n    def _create_densenet(num_classes, dropout_rate):\n        \n        model = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\n        \n        for param in model.parameters():\n            param.requires_grad = False\n        \n        for param in model.features[-2:].parameters():\n            param.requires_grad = True\n        \n        num_features = model.classifier.in_features\n        model.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(num_features, 512),\n            nn.ReLU(),\n            nn.Dropout(dropout_rate),\n            nn.Linear(512, num_classes))\n        \n        return model\n\n\n#5.2 Обучение\n\ndef train_and_evaluate_model(model_name, model, train_loader, valid_loader, \n                             device, num_epochs=10, learning_rate=1e-4):\n    \n    total_params = sum(p.numel() for p in model.parameters())\n    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n    \n    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer, mode='min', patience=3, factor=0.5)\n    criterion = nn.CrossEntropyLoss()\n    \n    history = {\n        'train_loss': [],\n        'train_acc': [],\n        'valid_loss': [],\n        'valid_acc': [],\n        'epoch_time': []}\n    \n    best_valid_acc = 0.0\n    best_model_state = None\n    \n    start_time = time.time()\n    \n    for epoch in range(num_epochs):\n        epoch_start = time.time()\n        \n        train_loss, train_acc = train_one_epoch(\n            model, train_loader, optimizer, criterion, \n            epoch, device, verbose=False)\n        \n        valid_loss, valid_acc = valid_one_epoch(\n            model, valid_loader, criterion, \n            epoch, device, verbose=False)\n        \n        scheduler.step(valid_loss)\n        \n        epoch_time = time.time() - epoch_start\n        history['train_loss'].append(train_loss)\n        history['train_acc'].append(train_acc)\n        history['valid_loss'].append(valid_loss)\n        history['valid_acc'].append(valid_acc)\n        history['epoch_time'].append(epoch_time)\n        \n        if valid_acc > best_valid_acc:\n            best_valid_acc = valid_acc\n            best_model_state = model.state_dict().copy()\n            torch.save(best_model_state, f'best_{model_name}.pth')\n        \n        print(f\"Epoch {epoch+1}/{num_epochs} ({epoch_time:.1f}s):\")\n        print(f\"Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.2f}%\")\n        print(f\"Valid Loss: {valid_loss:.4f}, Valid Acc: {valid_acc:.2f}%\")\n        print(f\"LR: {optimizer.param_groups[0]['lr']:.6f}\")\n    \n    total_time = time.time() - start_time\n    \n    print(f\"\\n Результаты {model_name.upper()}:\")\n    print(f\"- Лучшая точность: {best_valid_acc:.2f}%\")\n    print(f\"- Время обучения: {total_time:.1f} сек ({total_time/60:.1f} мин)\")\n    \n    return {\n        'model_name': model_name,\n        'best_valid_acc': best_valid_acc,\n        'total_time': total_time,\n        'history': history,\n        'total_params': total_params,\n        'trainable_params': trainable_params,\n        'best_model_state': best_model_state}\n\n\n#5.3 Запуск моделей\n\nmodels_to_test = [\n    ('resnet50', 'ResNet50'),\n    ('efficientnet', 'EfficientNet-B0'),\n    ('densenet', 'DenseNet121')]\n\nexperiment_results = {}\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Устройство: {device}\")\n\nnum_epochs = 7  \nlearning_rate = 1e-4\n\nfor model_key, model_display in models_to_test:\n    print(f\"Обучение модели: {model_display}\")\n    \n    model = ModelFactory.create_model(\n        model_name=model_key,\n        num_classes=num_classes,\n        dropout_rate=0.3)\n    model = model.to(device)\n    \n    result = train_and_evaluate_model(\n        model_name=model_key,\n        model=model,\n        train_loader=train_loader,\n        valid_loader=valid_loader,\n        device=device,\n        num_epochs=num_epochs,\n        learning_rate=learning_rate)\n    \n    experiment_results[model_key] = result\n\n\n#5.4 Результаты\n\ncomparison_data = []\n\nfor model_key, result in experiment_results.items():\n    display_name = {\n        'resnet50': 'ResNet50',\n        'efficientnet': 'EfficientNet-B0',\n        'densenet': 'DenseNet121'}[model_key]\n    \n    history = result['history']\n    \n    comparison_data.append({\n        'Модель': display_name,\n        'Лучшая точность в %': round(result['best_valid_acc'], 2),\n        'Средняя точность Train в %': round(np.mean(history['train_acc']), 2),\n        'Средняя точность Valid в %': round(np.mean(history['valid_acc']), 2),\n        'Средняя потеря Train': round(np.mean(history['train_loss']), 4),\n        'Средняя потеря Valid': round(np.mean(history['valid_loss']), 4),\n        'Стабильность (σ Train)': round(np.std(history['train_acc']), 2),\n        'Стабильность (σ Valid)': round(np.std(history['valid_acc']), 2),\n        'Разница Train/Valid в %': round(np.mean(history['train_acc']) - np.mean(history['valid_acc']), 2),\n        'Среднее время эпохи (сек)': round(np.mean(history['epoch_time']), 1),\n        'Общее время (сек)': round(result['total_time'], 1),\n        'Общее время (мин)': round(result['total_time'] / 60, 1),\n        'Всего параметров': result['total_params'],\n        'Обучаемых параметров': result['trainable_params'],\n        'Процент обучаемых в %': round(result['trainable_params'] / result['total_params'] * 100, 1)\n    })\n\ndf_comparison = pd.DataFrame(comparison_data)\ndf_comparison = df_comparison.sort_values('Лучшая точность в %', ascending=False).reset_index(drop=True)\n\nprint(\"\\n Таблица сравнения:\")\nprint(df_comparison.to_string(index=False))\n\nexcel_filename = f'model_comparison.xlsx'\n\nwith pd.ExcelWriter(excel_filename, engine='openpyxl') as writer:\n    df_comparison.to_excel(writer, sheet_name='Сравнение моделей', index=False)\n    \n    for model_key, result in experiment_results.items():\n        display_name = {\n            'resnet50': 'ResNet50',\n            'efficientnet': 'EfficientNet-B0',\n            'densenet': 'DenseNet121'}[model_key]\n        \n        history = result['history']\n        history_df = pd.DataFrame({\n            'Эпоха': range(1, len(history['train_acc']) + 1),\n            'Train Loss': history['train_loss'],\n            'Train Accuraccy в %': history['train_acc'],\n            'Valid Loss': history['valid_loss'],\n            'Valid Accuraccy в %': history['valid_acc'],\n            'Время эпохи (сек)': history['epoch_time']})\n        history_df.to_excel(writer, sheet_name=f'{display_name}_история', index=False)\n\nprint(f\"\\n Файл сохранен: {excel_filename}\")\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-27T21:13:26.938881Z","iopub.execute_input":"2026-07-27T21:13:26.939978Z","iopub.status.idle":"2026-07-27T21:22:29.044159Z","shell.execute_reply.started":"2026-07-27T21:13:26.939938Z","shell.execute_reply":"2026-07-27T21:22:29.043358Z"}},"outputs":[],"execution_count":null}]}