{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 本代码由 ChatGPT 生成，作为个人学习卷积神经网络（CNN）的demo。","metadata":{}},{"cell_type":"markdown","source":"# 1.导入必要的库\n- torch\n- torchvision\n- pandas\n- matplotlib\n- pillow\n- tqdm\n- ipykernel","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset, random_split\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom torch.cuda.amp import GradScaler, autocast\nfrom tqdm import tqdm\n\ntorch.backends.cudnn.benchmark = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:13:44.445739Z","iopub.execute_input":"2025-03-17T15:13:44.446011Z","iopub.status.idle":"2025-03-17T15:13:44.450731Z","shell.execute_reply.started":"2025-03-17T15:13:44.445989Z","shell.execute_reply":"2025-03-17T15:13:44.449715Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2.数据读取","metadata":{}},{"cell_type":"markdown","source":"### 2.1解压train","metadata":{}},{"cell_type":"code","source":"!sudo apt-get install p7zip-full\n!7z x \"/kaggle/input/cifar-10/train.7z\"","metadata":{"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2 定义train与标签路径","metadata":{}},{"cell_type":"code","source":"IMG_DIR = '/kaggle/working/train'\nLABELS_FILE = '/kaggle/input/cifar-10/trainLabels.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:20:53.264168Z","iopub.execute_input":"2025-03-17T15:20:53.264442Z","iopub.status.idle":"2025-03-17T15:20:53.268107Z","shell.execute_reply.started":"2025-03-17T15:20:53.264415Z","shell.execute_reply":"2025-03-17T15:20:53.267388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.3 自定义Dataset用于迭代读取图片","metadata":{}},{"cell_type":"code","source":"class CustomCIFAR10Dataset(Dataset):\n    def __init__(self, img_dir, labels_file, transform=None):\n        # 初始化数据集，加载图像目录和标签文件\n        self.img_dir = img_dir\n        self.transform = transform\n        self.labels = pd.read_csv(labels_file)  # 读取标签文件\n        self.labels.set_index('id', inplace=True)  # 将标签文件的索引设置为 'id'\n\n    def __len__(self):\n        # 返回数据集的大小\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        # 根据索引获取图像和标签\n        img_name = f\"{self.labels.index[idx]}.png\"  # 获取图像文件名\n        img_path = os.path.join(self.img_dir, img_name)  # 获取图像路径\n        image = Image.open(img_path).convert('RGB')  # 打开图像并转换为 RGB 模式\n        if self.transform:\n            image = self.transform(image)  # 如果有变换操作，则应用变换\n        label = self.labels.loc[self.labels.index[idx], 'label']  # 获取标签\n        label = self.label_to_index(label)  # 将标签转换为索引\n        return image, label  # 返回图像和标签\n\n    def label_to_index(self, label):\n        # 将标签转换为索引\n        label_map = {\n            'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4,\n            'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9\n        }\n        return label_map[label]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:20:53.269394Z","iopub.execute_input":"2025-03-17T15:20:53.269652Z","iopub.status.idle":"2025-03-17T15:20:53.284420Z","shell.execute_reply.started":"2025-03-17T15:20:53.269621Z","shell.execute_reply":"2025-03-17T15:20:53.283784Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.4 数据增强和训练集验证集构建","metadata":{}},{"cell_type":"code","source":"# 数据预处理\nnormalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\ntrain_transforms = transforms.Compose([\n    transforms.RandomResizedCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    normalize\n])\ntest_transforms = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    normalize\n])\n\n# 构造数据集\ndataset = CustomCIFAR10Dataset(img_dir=IMG_DIR, labels_file=LABELS_FILE, transform=train_transforms)\n\n# 划分训练集和验证集\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n\n# 使用不同的变换\ntrain_dataset.dataset.transform = train_transforms\nval_dataset.dataset.transform = test_transforms\n\n# 数据加载器\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True, pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=128, shuffle=False, pin_memory=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:20:53.285080Z","iopub.execute_input":"2025-03-17T15:20:53.285319Z","iopub.status.idle":"2025-03-17T15:20:53.327891Z","shell.execute_reply.started":"2025-03-17T15:20:53.285288Z","shell.execute_reply":"2025-03-17T15:20:53.327314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3.加载ResNet-18模型","metadata":{}},{"cell_type":"code","source":"# 检查是否有可用的 GPU，如果有则使用 GPU，否则使用 CPU\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# 加载预训练的 ResNet-18 模型\nmodel = torchvision.models.resnet18(pretrained=True)\n\n# 修改最后一层全连接层，使其输出 10 个类别（对应 CIFAR-10 数据集的 10 个类别）\nmodel.fc = nn.Linear(512, 10)\n\n# 将模型移动到指定设备（GPU 或 CPU）\nmodel = model.to(device)\n\n# 定义损失函数为交叉熵损失\ncriterion = nn.CrossEntropyLoss()\n\n# 定义优化器为随机梯度下降（SGD），学习率为 0.001，动量为 0.9\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n# 定义梯度缩放器，用于混合精度训练\nscaler = GradScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:20:53.328524Z","iopub.execute_input":"2025-03-17T15:20:53.328724Z","iopub.status.idle":"2025-03-17T15:20:53.571461Z","shell.execute_reply.started":"2025-03-17T15:20:53.328706Z","shell.execute_reply":"2025-03-17T15:20:53.570672Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4.训练","metadata":{}},{"cell_type":"code","source":"num_epochs = 4\ntrain_losses = []\ntrain_accuracies = []\nval_accuracies = []\n\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\")\n    for images, labels in progress_bar:\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        with autocast():\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        running_loss += loss.item()\n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n        progress_bar.set_postfix(loss=running_loss/(total/labels.size(0)), accuracy=100.*correct/total)\n    train_loss = running_loss / len(train_loader)\n    train_acc = 100. * correct / total\n    train_losses.append(train_loss)\n    train_accuracies.append(train_acc)\n\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n            with autocast():\n                outputs = model(images)\n            _, predicted = outputs.max(1)\n            total += labels.size(0)\n            correct += predicted.eq(labels).sum().item()\n    val_acc = 100. * correct / total\n    val_accuracies.append(val_acc)\n\n    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {train_loss:.4f}, Train Acc: {train_acc:.2f}%, Val Acc: {val_acc:.2f}%')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:20:53.572146Z","iopub.execute_input":"2025-03-17T15:20:53.572374Z","iopub.status.idle":"2025-03-17T15:31:44.408492Z","shell.execute_reply.started":"2025-03-17T15:20:53.572354Z","shell.execute_reply":"2025-03-17T15:31:44.407495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(range(1, num_epochs+1), train_losses, label='Train Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training Loss')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(range(1, num_epochs+1), train_accuracies, label='Train Accuracy')\nplt.plot(range(1, num_epochs+1), val_accuracies, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:31:58.364018Z","iopub.execute_input":"2025-03-17T15:31:58.364362Z","iopub.status.idle":"2025-03-17T15:31:58.795222Z","shell.execute_reply.started":"2025-03-17T15:31:58.364335Z","shell.execute_reply":"2025-03-17T15:31:58.794327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 保存模型\ntorch.save(model.state_dict(), 'resnet18_cifar10.pth')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:32:10.369829Z","iopub.execute_input":"2025-03-17T15:32:10.370299Z","iopub.status.idle":"2025-03-17T15:32:10.441704Z","shell.execute_reply.started":"2025-03-17T15:32:10.370230Z","shell.execute_reply":"2025-03-17T15:32:10.441047Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5.推理","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset, random_split\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom torch.cuda.amp import GradScaler, autocast\nfrom tqdm import tqdm\n\ntorch.backends.cudnn.benchmark = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:33:56.296437Z","iopub.execute_input":"2025-03-17T15:33:56.296750Z","iopub.status.idle":"2025-03-17T15:33:56.301368Z","shell.execute_reply.started":"2025-03-17T15:33:56.296727Z","shell.execute_reply":"2025-03-17T15:33:56.300604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !sudo apt-get install p7zip-full\n!7z x \"/kaggle/input/cifar-10/test.7z\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMG_DIR = '/kaggle/workingtest'\nSUBMISSION_FILE = './submission.csv'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomCIFAR10TestDataset(Dataset):\n    def __init__(self, img_dir, transform=None):\n        self.img_dir = img_dir\n        self.transform = transform\n        self.img_names = os.listdir(img_dir)\n\n    def __len__(self):\n        return len(self.img_names)\n\n    def __getitem__(self, idx):\n        img_name = self.img_names[idx]\n        img_path = os.path.join(self.img_dir, img_name)\n        image = Image.open(img_path).convert('RGB')  # 使用 PIL 图像\n        if self.transform:\n            image = self.transform(image)\n        return image, img_name\n\n# 数据预处理\ntest_transforms = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# 构造测试数据集\ntest_dataset = CustomCIFAR10TestDataset(img_dir=IMG_DIR, transform=test_transforms)\n\n# 数据加载器\ntest_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, pin_memory=True)\n\n# 定义标签映射\nindex_to_label = {\n    0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer',\n    5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'\n}\n\n# 加载模型\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = torchvision.models.resnet18(weights=None)\nmodel.fc = nn.Linear(512, 10)\nmodel.load_state_dict(torch.load('resnet18_cifar10.pth'))\nmodel = model.to(device)\n\n# 确保模型处于评估模式\nmodel.eval()\n\n# 存储预测结果\npredictions = []\n\nwith torch.no_grad():\n    for images, img_names in tqdm(test_loader, desc=\"Inference\"):\n        images = images.to(device)\n        outputs = model(images)\n        _, predicted = outputs.max(1)\n        predicted = predicted.cpu().numpy()\n        for img_name, pred in zip(img_names, predicted):\n            label = index_to_label[pred]\n            predictions.append((img_name, label))\n\n# 保存预测结果到 submission.csv\nsubmission_df = pd.DataFrame(predictions, columns=['id', 'label'])\nsubmission_df['id'] = submission_df['id'].str.replace('.png', '')\n# submission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission id转为int 按照id排序 保存到'../data/cifar-10/submission.csv'\nsubmission_df['id'] = submission_df['id'].astype(int)\nsubmission_df = submission_df.sort_values('id')\nsubmission_df.to_csv(SUBMISSION_FILE, index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}