{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 图像分类\n**廖星裕**","metadata":{}},{"cell_type":"code","source":"pip install d2l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T02:32:48.000798Z","iopub.execute_input":"2025-06-24T02:32:48.001415Z","iopub.status.idle":"2025-06-24T02:33:08.689004Z","shell.execute_reply.started":"2025-06-24T02:32:48.001381Z","shell.execute_reply":"2025-06-24T02:33:08.688238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T02:33:59.899948Z","iopub.execute_input":"2025-06-24T02:33:59.900678Z","iopub.status.idle":"2025-06-24T02:34:04.756312Z","shell.execute_reply.started":"2025-06-24T02:33:59.900650Z","shell.execute_reply":"2025-06-24T02:34:04.755363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install efficientnet_pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T02:35:04.217718Z","iopub.execute_input":"2025-06-24T02:35:04.218520Z","iopub.status.idle":"2025-06-24T02:36:22.610012Z","shell.execute_reply.started":"2025-06-24T02:35:04.218489Z","shell.execute_reply":"2025-06-24T02:36:22.609062Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 一、数据预处理","metadata":{}},{"cell_type":"markdown","source":"### 导入库和设置随机种子","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms, datasets\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\nfrom torchvision.models import resnet18\nimport numpy as np\nimport pandas as pd\n\n# 设置随机种子保证可重复性\ntorch.manual_seed(42)\nnp.random.seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:29:28.055038Z","iopub.execute_input":"2025-06-24T03:29:28.055715Z","iopub.status.idle":"2025-06-24T03:29:28.062260Z","shell.execute_reply.started":"2025-06-24T03:29:28.055686Z","shell.execute_reply":"2025-06-24T03:29:28.061505Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 数据准备","metadata":{}},{"cell_type":"code","source":"# 1. 数据预处理\ntrain_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.RandomResizedCrop(224, scale=(0.64, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\ntest_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# 加载CIFAR10数据集\ntrain_data = datasets.CIFAR10(root='./data', train=True, download=True, transform=train_transform)\ntest_data = datasets.CIFAR10(root='./data', train=False, download=True, transform=test_transform)\n\n# 划分训练集和验证集\nn = len(train_data)\nindices = torch.randperm(n)\nsplit = int(n * 0.9)\ntrain_subset = torch.utils.data.Subset(train_data, indices[:split])\nvalid_subset = torch.utils.data.Subset(train_data, indices[split:])\n\nbatch_size = 64\ntrain_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\nvalid_loader = DataLoader(valid_subset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\ntest_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n\nclass_names = ['Airplane', 'Car', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Boat', 'Truck']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:29:47.392227Z","iopub.execute_input":"2025-06-24T03:29:47.392935Z","iopub.status.idle":"2025-06-24T03:29:49.164864Z","shell.execute_reply.started":"2025-06-24T03:29:47.392905Z","shell.execute_reply":"2025-06-24T03:29:49.164098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 数据集信息查看","metadata":{}},{"cell_type":"code","source":"print(\"\\n查看数据集信息...\")\nprint(f\"训练集样本数: {len(train_subset)}\")\nprint(f\"验证集样本数: {len(valid_subset)}\")\nprint(f\"测试集样本数: {len(test_data)}\")\nprint(\"\\n类别分布:\")\n\n# 统计训练集类别分布\ntrain_labels = [train_data.targets[i] for i in train_subset.indices]\ntrain_class_counts = {class_name: train_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"训练集:\", train_class_counts)\n\n# 统计验证集类别分布\nvalid_labels = [train_data.targets[i] for i in valid_subset.indices]\nvalid_class_counts = {class_name: valid_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"验证集:\", valid_class_counts)\n\n# 统计测试集类别分布\ntest_labels = test_data.targets\ntest_class_counts = {class_name: test_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"测试集:\", test_class_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:29:52.804318Z","iopub.execute_input":"2025-06-24T03:29:52.804587Z","iopub.status.idle":"2025-06-24T03:29:53.213007Z","shell.execute_reply.started":"2025-06-24T03:29:52.804568Z","shell.execute_reply":"2025-06-24T03:29:53.212408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 图像处理对比","metadata":{}},{"cell_type":"code","source":"# 添加查看处理前后图像的代码\ndef denormalize(tensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]):\n    \"\"\"反归一化图像以便显示\"\"\"\n    for t, m, s in zip(tensor, mean, std):\n        t.mul_(s).add_(m)\n    return tensor\n\nprint(\"\\n显示原始图像和处理后图像对比...\")\n# 获取一个批次的原始图像和处理后图像\noriginal_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.CenterCrop(224),\n    transforms.ToTensor()\n])\n\noriginal_data = datasets.CIFAR10(root='./data', train=True, download=True, transform=original_transform)\nsample_indices = torch.randperm(len(original_data))[:8]  # 随机选择8个样本\n\nplt.figure(figsize=(12, 6))\nfor i, idx in enumerate(sample_indices):\n    # 原始图像\n    original_img, label = original_data[idx]\n    plt.subplot(2, 8, i+1)\n    plt.imshow(original_img.permute(1, 2, 0))\n    plt.title(f\"Original\\n{class_names[label]}\")\n    plt.axis('off')\n    \n    # 处理后的图像\n    processed_img, _ = train_data[idx]\n    processed_img = denormalize(processed_img)  # 反归一化\n    plt.subplot(2, 8, i+9)\n    plt.imshow(processed_img.permute(1, 2, 0))\n    plt.title(f\"Processed\\n{class_names[label]}\")\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:29:59.203799Z","iopub.execute_input":"2025-06-24T03:29:59.204118Z","iopub.status.idle":"2025-06-24T03:30:01.192325Z","shell.execute_reply.started":"2025-06-24T03:29:59.204097Z","shell.execute_reply":"2025-06-24T03:30:01.191468Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 二、模型训练","metadata":{}},{"cell_type":"markdown","source":"### 模型准备","metadata":{}},{"cell_type":"code","source":"# 2. 模型准备\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"使用设备: {device}\")\n\n# 准备EfficientNet模型\nprint(\"\\n准备EfficientNet模型...\")\neffnet = EfficientNet.from_pretrained('efficientnet-b0')\n# 冻结所有层\nfor param in effnet.parameters():\n    param.requires_grad = False\n# 解冻最后几层\nfor param in effnet._conv_head.parameters():\n    param.requires_grad = True\nfor param in effnet._bn1.parameters():\n    param.requires_grad = True\nfor param in effnet._fc.parameters():\n    param.requires_grad = True\n# 修改最后的全连接层\neffnet._fc = nn.Linear(effnet._fc.in_features, 10)\neffnet = effnet.to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:30:28.328719Z","iopub.execute_input":"2025-06-24T03:30:28.329067Z","iopub.status.idle":"2025-06-24T03:30:28.684769Z","shell.execute_reply.started":"2025-06-24T03:30:28.329046Z","shell.execute_reply":"2025-06-24T03:30:28.684185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 准备ResNet模型\nprint(\"\\n准备ResNet模型...\")\nresnet = resnet18(pretrained=True)\n# 冻结所有层\nfor param in resnet.parameters():\n    param.requires_grad = False\n# 解冻最后几层\nfor param in resnet.layer4.parameters():\n    param.requires_grad = True\nfor param in resnet.fc.parameters():\n    param.requires_grad = True\n# 修改最后的全连接层\nresnet.fc = nn.Linear(resnet.fc.in_features, 10)\nresnet = resnet.to(device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 训练配置","metadata":{}},{"cell_type":"code","source":"# 3. 训练配置\ncriterion = nn.CrossEntropyLoss()\nepochs = 10\neffnet_train_losses = []\neffnet_train_accs = []\neffnet_val_accs = []\nresnet_train_losses = []\nresnet_train_accs = []\nresnet_val_accs = []\nfinal_train_losses = []\nfinal_train_accs = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:30:39.962313Z","iopub.execute_input":"2025-06-24T03:30:39.963020Z","iopub.status.idle":"2025-06-24T03:30:39.968467Z","shell.execute_reply.started":"2025-06-24T03:30:39.962991Z","shell.execute_reply":"2025-06-24T03:30:39.967676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 训练EfficientNet","metadata":{}},{"cell_type":"code","source":"# 4. 训练EfficientNet\nprint(\"\\n开始训练EfficientNet...\")\neffnet_optimizer = optim.Adam(filter(lambda p: p.requires_grad, effnet.parameters()), lr=1e-3, weight_decay=5e-4)\neffnet_scheduler = optim.lr_scheduler.StepLR(effnet_optimizer, step_size=2, gamma=0.9)\nbest_effnet_acc = 0.0\n\nfor epoch in range(epochs):\n    effnet.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(train_loader, desc=f\"EfficientNet 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        effnet_optimizer.zero_grad(set_to_none=True)\n        outputs = effnet(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        effnet_optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    effnet_scheduler.step()\n    train_loss = total_loss / len(train_loader)\n    train_acc = correct / total\n    \n    # 记录训练损失和准确率\n    effnet_train_losses.append(train_loss)\n    effnet_train_accs.append(train_acc)\n    \n    # 验证\n    effnet.eval()\n    val_correct = 0\n    val_total = 0\n    with torch.no_grad():\n        for inputs, labels in valid_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = effnet(inputs)\n            _, predicted = outputs.max(1)\n            val_correct += predicted.eq(labels).sum().item()\n            val_total += labels.size(0)\n    \n    val_acc = val_correct / val_total\n    effnet_val_accs.append(val_acc)\n    \n    if val_acc > best_effnet_acc:\n        best_effnet_acc = val_acc\n        torch.save(effnet.state_dict(), 'best_effnet.pth')\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}, 验证准确率: {val_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:30:44.612670Z","iopub.execute_input":"2025-06-24T03:30:44.612984Z","iopub.status.idle":"2025-06-24T03:51:39.507126Z","shell.execute_reply.started":"2025-06-24T03:30:44.612933Z","shell.execute_reply":"2025-06-24T03:51:39.506109Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 训练ResNet","metadata":{}},{"cell_type":"code","source":"# 5. 训练ResNet\nprint(\"\\n开始训练ResNet...\")\nresnet_optimizer = optim.Adam(filter(lambda p: p.requires_grad, resnet.parameters()), lr=1e-3, weight_decay=5e-4)\nresnet_scheduler = optim.lr_scheduler.StepLR(resnet_optimizer, step_size=2, gamma=0.9)\nbest_resnet_acc = 0.0\n\nfor epoch in range(epochs):\n    resnet.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(train_loader, desc=f\"ResNet 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        resnet_optimizer.zero_grad(set_to_none=True)\n        outputs = resnet(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        resnet_optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    resnet_scheduler.step()\n    train_loss = total_loss / len(train_loader)\n    train_acc = correct / total\n    \n    # 记录训练损失和准确率\n    resnet_train_losses.append(train_loss)\n    resnet_train_accs.append(train_acc)\n    \n    # 验证\n    resnet.eval()\n    val_correct = 0\n    val_total = 0\n    with torch.no_grad():\n        for inputs, labels in valid_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = resnet(inputs)\n            _, predicted = outputs.max(1)\n            val_correct += predicted.eq(labels).sum().item()\n            val_total += labels.size(0)\n    \n    val_acc = val_correct / val_total\n    resnet_val_accs.append(val_acc)\n    \n    if val_acc > best_resnet_acc:\n        best_resnet_acc = val_acc\n        torch.save(resnet.state_dict(), 'best_resnet.pth')\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}, 验证准确率: {val_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T03:57:19.339690Z","iopub.execute_input":"2025-06-24T03:57:19.340068Z","iopub.status.idle":"2025-06-24T04:16:51.606072Z","shell.execute_reply.started":"2025-06-24T03:57:19.340023Z","shell.execute_reply":"2025-06-24T04:16:51.604993Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 绘制训练曲线","metadata":{}},{"cell_type":"code","source":"# 绘制EfficientNet和ResNet的训练曲线\nplt.figure(figsize=(12, 5))\n\n# 损失曲线\nplt.subplot(1, 2, 1)\nplt.plot(effnet_train_losses, label='EfficientNet Training loss')\nplt.plot(resnet_train_losses, label='ResNet Training loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training loss curve')\nplt.legend()\n\n# 准确率曲线\nplt.subplot(1, 2, 2)\nplt.plot(effnet_train_accs, label='EfficientNet Training accuracy')\nplt.plot(effnet_val_accs, label='EfficientNet Verify accuracy', linestyle='--')\nplt.plot(resnet_train_accs, label='ResNet Training accuracy')\nplt.plot(resnet_val_accs, label='ResNet Verify accuracy', linestyle='--')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Training and Validation Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:17:09.041606Z","iopub.execute_input":"2025-06-24T04:17:09.041887Z","iopub.status.idle":"2025-06-24T04:17:09.435808Z","shell.execute_reply.started":"2025-06-24T04:17:09.041864Z","shell.execute_reply":"2025-06-24T04:17:09.435017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 选择最优模型","metadata":{}},{"cell_type":"code","source":"# 6. 选择最优模型\nif best_effnet_acc > best_resnet_acc:\n    best_model = effnet\n    best_model.load_state_dict(torch.load('best_effnet.pth'))\n    print(\"\\n选择EfficientNet作为最优模型\")\nelse:\n    best_model = resnet\n    best_model.load_state_dict(torch.load('best_resnet.pth'))\n    print(\"\\n选择ResNet作为最优模型\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:20:11.330913Z","iopub.execute_input":"2025-06-24T04:20:11.331727Z","iopub.status.idle":"2025-06-24T04:20:11.401181Z","shell.execute_reply.started":"2025-06-24T04:20:11.331698Z","shell.execute_reply":"2025-06-24T04:20:11.400325Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 完整训练集训练","metadata":{}},{"cell_type":"code","source":"# 7. 在完整训练集上训练最优模型\nprint(\"\\n在完整训练集上训练最优模型...\")\nfull_train_loader = DataLoader(\n    torch.utils.data.ConcatDataset([train_loader.dataset, valid_loader.dataset]),\n    batch_size=64, shuffle=True, num_workers=4, pin_memory=True\n)\n\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, best_model.parameters()), lr=1e-3, weight_decay=5e-4)\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.9)\n\nfor epoch in range(10):\n    best_model.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(full_train_loader, desc=f\"完整训练 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        optimizer.zero_grad(set_to_none=True)\n        outputs = best_model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    scheduler.step()\n    train_loss = total_loss / len(full_train_loader)\n    train_acc = correct / total\n    \n    # 记录最终模型的训练损失和准确率\n    final_train_losses.append(train_loss)\n    final_train_accs.append(train_acc)\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:20:23.737551Z","iopub.execute_input":"2025-06-24T04:20:23.738258Z","iopub.status.idle":"2025-06-24T04:39:56.522205Z","shell.execute_reply.started":"2025-06-24T04:20:23.738235Z","shell.execute_reply":"2025-06-24T04:39:56.521234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 绘制最终模型训练曲线","metadata":{}},{"cell_type":"code","source":"# 绘制最终模型的训练曲线\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(final_train_losses, label='Training loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('The final model training loss curve')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(final_train_accs, label='Training accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('The final model training accuracy curve')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n\n# 保存最终模型\ntorch.save(best_model.state_dict(), 'final_best_model.pth')\nprint(\"已保存最终模型: final_best_model.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:41:02.895314Z","iopub.execute_input":"2025-06-24T04:41:02.896295Z","iopub.status.idle":"2025-06-24T04:41:03.412340Z","shell.execute_reply.started":"2025-06-24T04:41:02.896264Z","shell.execute_reply":"2025-06-24T04:41:03.411709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 三、模型评估","metadata":{}},{"cell_type":"code","source":"# 8. 在测试集上评估\nprint(\"\\n在测试集上评估最优模型...\")\nbest_model.eval()\ntest_correct = 0\ntest_total = 0\nall_preds = []\nall_labels = []\nall_probs = []\nall_images = []\n\nwith torch.no_grad():\n    for inputs, labels in test_loader:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        outputs = best_model(inputs)\n        probs = torch.nn.functional.softmax(outputs, dim=1)\n        _, predicted = outputs.max(1)\n        test_correct += predicted.eq(labels).sum().item()\n        test_total += labels.size(0)\n        all_preds.extend(predicted.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n        all_probs.extend(probs.cpu().numpy())\n        all_images.extend(inputs.cpu().numpy())\n\ntest_acc = test_correct / test_total\nprint(f\"测试准确率: {test_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:41:10.167793Z","iopub.execute_input":"2025-06-24T04:41:10.168114Z","iopub.status.idle":"2025-06-24T04:41:26.305123Z","shell.execute_reply.started":"2025-06-24T04:41:10.168092Z","shell.execute_reply":"2025-06-24T04:41:26.304269Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 输出分类报告和混淆矩阵","metadata":{}},{"cell_type":"code","source":"# 10. 输出分类报告和混淆矩阵\nprint(\"\\n分类报告:\")\nprint(classification_report(all_labels, all_preds, target_names=class_names))\n\n# 显示混淆矩阵\nplt.figure(figsize=(10, 8))\nsns.heatmap(confusion_matrix(all_labels, all_preds), \n            annot=True, fmt='d', cmap='Blues',\n            xticklabels=class_names, yticklabels=class_names)\nplt.title(\"Confusion Matrix\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:48:34.925746Z","iopub.execute_input":"2025-06-24T04:48:34.926292Z","iopub.status.idle":"2025-06-24T04:48:35.335406Z","shell.execute_reply.started":"2025-06-24T04:48:34.926270Z","shell.execute_reply":"2025-06-24T04:48:35.334711Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 保存预测结果","metadata":{}},{"cell_type":"code","source":"# 9. 保存预测结果\nprint(\"\\n保存预测结果...\")\nresults = []\nfor i in range(len(all_preds)):\n    results.append({\n        '真实标签': class_names[all_labels[i]],\n        '预测标签': class_names[all_preds[i]],\n        '预测概率': max(all_probs[i]),\n        '是否正确': all_labels[i] == all_preds[i]\n    })\n\nresults_df = pd.DataFrame(results)\nresults_df.to_csv('predictions_results.csv', index=False, encoding='utf-8-sig')\nprint(\"预测结果已保存到 predictions_results.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T04:48:31.647207Z","iopub.execute_input":"2025-06-24T04:48:31.647892Z","iopub.status.idle":"2025-06-24T04:48:31.717251Z","shell.execute_reply.started":"2025-06-24T04:48:31.647862Z","shell.execute_reply":"2025-06-24T04:48:31.716575Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 代码汇总","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms, datasets\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\nfrom torchvision.models import resnet18\nimport numpy as np\nimport pandas as pd\n\n# 设置随机种子保证可重复性\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# 1. 数据准备\nprint(\"准备数据...\")\ntrain_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.RandomResizedCrop(224, scale=(0.64, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\ntest_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# 加载CIFAR10数据集\ntrain_data = datasets.CIFAR10(root='./data', train=True, download=True, transform=train_transform)\ntest_data = datasets.CIFAR10(root='./data', train=False, download=True, transform=test_transform)\n\n# 划分训练集和验证集\nn = len(train_data)\nindices = torch.randperm(n)\nsplit = int(n * 0.9)\ntrain_subset = torch.utils.data.Subset(train_data, indices[:split])\nvalid_subset = torch.utils.data.Subset(train_data, indices[split:])\n\nbatch_size = 64\ntrain_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\nvalid_loader = DataLoader(valid_subset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\ntest_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n\nclass_names = ['Airplane', 'Car', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Boat', 'Truck']\n\n# 添加数据查看代码\nprint(\"\\n查看数据集信息...\")\nprint(f\"训练集样本数: {len(train_subset)}\")\nprint(f\"验证集样本数: {len(valid_subset)}\")\nprint(f\"测试集样本数: {len(test_data)}\")\nprint(\"\\n类别分布:\")\n\n# 统计训练集类别分布\ntrain_labels = [train_data.targets[i] for i in train_subset.indices]\ntrain_class_counts = {class_name: train_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"训练集:\", train_class_counts)\n\n# 统计验证集类别分布\nvalid_labels = [train_data.targets[i] for i in valid_subset.indices]\nvalid_class_counts = {class_name: valid_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"验证集:\", valid_class_counts)\n\n# 统计测试集类别分布\ntest_labels = test_data.targets\ntest_class_counts = {class_name: test_labels.count(i) for i, class_name in enumerate(class_names)}\nprint(\"测试集:\", test_class_counts)\n\n# 添加查看处理前后图像的代码\ndef denormalize(tensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]):\n    \"\"\"反归一化图像以便显示\"\"\"\n    for t, m, s in zip(tensor, mean, std):\n        t.mul_(s).add_(m)\n    return tensor\n\nprint(\"\\n显示原始图像和处理后图像对比...\")\n# 获取一个批次的原始图像和处理后图像\noriginal_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.CenterCrop(224),\n    transforms.ToTensor()\n])\n\noriginal_data = datasets.CIFAR10(root='./data', train=True, download=True, transform=original_transform)\nsample_indices = torch.randperm(len(original_data))[:8]  # 随机选择8个样本\n\nplt.figure(figsize=(12, 6))\nfor i, idx in enumerate(sample_indices):\n    # 原始图像\n    original_img, label = original_data[idx]\n    plt.subplot(2, 8, i+1)\n    plt.imshow(original_img.permute(1, 2, 0))\n    plt.title(f\"Original\\n{class_names[label]}\")\n    plt.axis('off')\n    \n    # 处理后的图像\n    processed_img, _ = train_data[idx]\n    processed_img = denormalize(processed_img)  # 反归一化\n    plt.subplot(2, 8, i+9)\n    plt.imshow(processed_img.permute(1, 2, 0))\n    plt.title(f\"Processed\\n{class_names[label]}\")\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()\n\n# 2. 模型准备\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"使用设备: {device}\")\n\n# 准备EfficientNet模型\nprint(\"\\n准备EfficientNet模型...\")\neffnet = EfficientNet.from_pretrained('efficientnet-b0')\n# 冻结所有层\nfor param in effnet.parameters():\n    param.requires_grad = False\n# 解冻最后几层\nfor param in effnet._conv_head.parameters():\n    param.requires_grad = True\nfor param in effnet._bn1.parameters():\n    param.requires_grad = True\nfor param in effnet._fc.parameters():\n    param.requires_grad = True\n# 修改最后的全连接层\neffnet._fc = nn.Linear(effnet._fc.in_features, 10)\neffnet = effnet.to(device)\n\n# 准备ResNet模型\nprint(\"\\n准备ResNet模型...\")\nresnet = resnet18(pretrained=True)\n# 冻结所有层\nfor param in resnet.parameters():\n    param.requires_grad = False\n# 解冻最后几层\nfor param in resnet.layer4.parameters():\n    param.requires_grad = True\nfor param in resnet.fc.parameters():\n    param.requires_grad = True\n# 修改最后的全连接层\nresnet.fc = nn.Linear(resnet.fc.in_features, 10)\nresnet = resnet.to(device)\n\n# 3. 训练配置\ncriterion = nn.CrossEntropyLoss()\nepochs = 5\n\n# 初始化用于存储损失和准确率的列表\neffnet_train_losses = []\neffnet_train_accs = []\neffnet_val_accs = []\n\nresnet_train_losses = []\nresnet_train_accs = []\nresnet_val_accs = []\n\nfinal_train_losses = []\nfinal_train_accs = []\n\n# 4. 训练EfficientNet\nprint(\"\\n开始训练EfficientNet...\")\neffnet_optimizer = optim.Adam(filter(lambda p: p.requires_grad, effnet.parameters()), lr=1e-3, weight_decay=5e-4)\neffnet_scheduler = optim.lr_scheduler.StepLR(effnet_optimizer, step_size=2, gamma=0.9)\nbest_effnet_acc = 0.0\n\nfor epoch in range(epochs):\n    effnet.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(train_loader, desc=f\"EfficientNet 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        effnet_optimizer.zero_grad(set_to_none=True)\n        outputs = effnet(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        effnet_optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    effnet_scheduler.step()\n    train_loss = total_loss / len(train_loader)\n    train_acc = correct / total\n    \n    # 记录训练损失和准确率\n    effnet_train_losses.append(train_loss)\n    effnet_train_accs.append(train_acc)\n    \n    # 验证\n    effnet.eval()\n    val_correct = 0\n    val_total = 0\n    with torch.no_grad():\n        for inputs, labels in valid_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = effnet(inputs)\n            _, predicted = outputs.max(1)\n            val_correct += predicted.eq(labels).sum().item()\n            val_total += labels.size(0)\n    \n    val_acc = val_correct / val_total\n    effnet_val_accs.append(val_acc)\n    \n    if val_acc > best_effnet_acc:\n        best_effnet_acc = val_acc\n        torch.save(effnet.state_dict(), 'best_effnet.pth')\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}, 验证准确率: {val_acc:.4f}\")\n\n# 5. 训练ResNet\nprint(\"\\n开始训练ResNet...\")\nresnet_optimizer = optim.Adam(filter(lambda p: p.requires_grad, resnet.parameters()), lr=1e-3, weight_decay=5e-4)\nresnet_scheduler = optim.lr_scheduler.StepLR(resnet_optimizer, step_size=2, gamma=0.9)\nbest_resnet_acc = 0.0\n\nfor epoch in range(epochs):\n    resnet.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(train_loader, desc=f\"ResNet 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        resnet_optimizer.zero_grad(set_to_none=True)\n        outputs = resnet(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        resnet_optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    resnet_scheduler.step()\n    train_loss = total_loss / len(train_loader)\n    train_acc = correct / total\n    \n    # 记录训练损失和准确率\n    resnet_train_losses.append(train_loss)\n    resnet_train_accs.append(train_acc)\n    \n    # 验证\n    resnet.eval()\n    val_correct = 0\n    val_total = 0\n    with torch.no_grad():\n        for inputs, labels in valid_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = resnet(inputs)\n            _, predicted = outputs.max(1)\n            val_correct += predicted.eq(labels).sum().item()\n            val_total += labels.size(0)\n    \n    val_acc = val_correct / val_total\n    resnet_val_accs.append(val_acc)\n    \n    if val_acc > best_resnet_acc:\n        best_resnet_acc = val_acc\n        torch.save(resnet.state_dict(), 'best_resnet.pth')\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}, 验证准确率: {val_acc:.4f}\")\n\n# 绘制EfficientNet和ResNet的训练曲线\nplt.figure(figsize=(12, 5))\n\n# 损失曲线\nplt.subplot(1, 2, 1)\nplt.plot(effnet_train_losses, label='EfficientNet Training loss')\nplt.plot(resnet_train_losses, label='ResNet Training loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training loss curve')\nplt.legend()\n\n# 准确率曲线\nplt.subplot(1, 2, 2)\nplt.plot(effnet_train_accs, label='EfficientNet Training accuracy')\nplt.plot(effnet_val_accs, label='EfficientNet Verify accuracy', linestyle='--')\nplt.plot(resnet_train_accs, label='ResNet Training accuracy')\nplt.plot(resnet_val_accs, label='ResNet Verify accuracy', linestyle='--')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Training and Validation Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n\n# 6. 选择最优模型\nif best_effnet_acc > best_resnet_acc:\n    best_model = effnet\n    best_model.load_state_dict(torch.load('best_effnet.pth'))\n    print(\"\\n选择EfficientNet作为最优模型\")\nelse:\n    best_model = resnet\n    best_model.load_state_dict(torch.load('best_resnet.pth'))\n    print(\"\\n选择ResNet作为最优模型\")\n\n# 7. 在完整训练集上训练最优模型\nprint(\"\\n在完整训练集上训练最优模型...\")\nfull_train_loader = DataLoader(\n    torch.utils.data.ConcatDataset([train_loader.dataset, valid_loader.dataset]),\n    batch_size=64, shuffle=True, num_workers=4, pin_memory=True\n)\n\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, best_model.parameters()), lr=1e-3, weight_decay=5e-4)\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.9)\n\nfor epoch in range(10):\n    best_model.train()\n    total_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for inputs, labels in tqdm(full_train_loader, desc=f\"完整训练 第 {epoch+1} 轮\"):\n        inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        \n        optimizer.zero_grad(set_to_none=True)\n        outputs = best_model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n        _, predicted = outputs.max(1)\n        correct += predicted.eq(labels).sum().item()\n        total += labels.size(0)\n    \n    scheduler.step()\n    train_loss = total_loss / len(full_train_loader)\n    train_acc = correct / total\n    \n    # 记录最终模型的训练损失和准确率\n    final_train_losses.append(train_loss)\n    final_train_accs.append(train_acc)\n    \n    print(f\"第 {epoch+1} 轮: 损失: {train_loss:.4f}, 训练准确率: {train_acc:.4f}\")\n\n# 绘制最终模型的训练曲线\nplt.figure(figsize=(12, 5))\n\nplt.subplot(1, 2, 1)\nplt.plot(final_train_losses, label='Training loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('The final model training loss curve')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(final_train_accs, label='Training accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('The final model training accuracy curve')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n\n# 保存最终模型\ntorch.save(best_model.state_dict(), 'final_best_model.pth')\nprint(\"已保存最终模型: final_best_model.pth\")\n\n# 8. 在测试集上评估\nprint(\"\\n在测试集上评估最优模型...\")\nbest_model.eval()\ntest_correct = 0\ntest_total = 0\nall_preds = []\nall_labels = []\nall_probs = []\nall_images = []\n\nwith torch.no_grad():\n    for inputs, labels in test_loader:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        outputs = best_model(inputs)\n        probs = torch.nn.functional.softmax(outputs, dim=1)\n        _, predicted = outputs.max(1)\n        test_correct += predicted.eq(labels).sum().item()\n        test_total += labels.size(0)\n        all_preds.extend(predicted.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n        all_probs.extend(probs.cpu().numpy())\n        all_images.extend(inputs.cpu().numpy())\n\ntest_acc = test_correct / test_total\nprint(f\"测试准确率: {test_acc:.4f}\")\n\n# 9. 保存预测结果\nprint(\"\\n保存预测结果...\")\nresults = []\nfor i in range(len(all_preds)):\n    results.append({\n        '真实标签': class_names[all_labels[i]],\n        '预测标签': class_names[all_preds[i]],\n        '预测概率': max(all_probs[i]),\n        '是否正确': all_labels[i] == all_preds[i]\n    })\n\nresults_df = pd.DataFrame(results)\nresults_df.to_csv('predictions_results.csv', index=False, encoding='utf-8-sig')\nprint(\"预测结果已保存到 predictions_results.csv\")\n\n# 10. 输出分类报告和混淆矩阵\nprint(\"\\n分类报告:\")\nprint(classification_report(all_labels, all_preds, target_names=class_names))\n\n# 显示混淆矩阵\nplt.figure(figsize=(10, 8))\nsns.heatmap(confusion_matrix(all_labels, all_preds), \n            annot=True, fmt='d', cmap='Blues',\n            xticklabels=class_names, yticklabels=class_names)\nplt.title(\"Confusion Matrix\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T02:36:47.367970Z","iopub.execute_input":"2025-06-24T02:36:47.368705Z","iopub.status.idle":"2025-06-24T03:16:43.623798Z","shell.execute_reply.started":"2025-06-24T02:36:47.368675Z","shell.execute_reply":"2025-06-24T03:16:43.622804Z"}},"outputs":[],"execution_count":null}]}