{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3362,"databundleVersionId":31148,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"导入数据库","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:31:29.656112Z","iopub.execute_input":"2024-11-01T02:31:29.656622Z","iopub.status.idle":"2024-11-01T02:31:41.261834Z","shell.execute_reply.started":"2024-11-01T02:31:29.656572Z","shell.execute_reply":"2024-11-01T02:31:41.260657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#导入数据库\n# Basic\nimport zipfile\nimport os\nfrom os import makedirs\nfrom os import listdir\nfrom shutil import copyfile\nfrom random import seed\nfrom random import random\nimport numpy as np\nimport pandas as pd\nimport imghdr\nimport random\n# visuals\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# Scikit-learn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,confusion_matrix,ConfusionMatrixDisplay\n\n# Pytorch\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torch import optim\nfrom torch.autograd import Variable\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\nfrom torchvision.models import  squeezenet1_1\nfrom torch.optim import Adam\nfrom re import S\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm, trange\nimport time\nfrom torch.utils.tensorboard import SummaryWriter\nimport datetime\nimport shutil\nfrom torch.utils import data\nfrom torchsummary import summary\nfrom torchvision import models\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:31:41.263887Z","iopub.execute_input":"2024-11-01T02:31:41.264189Z","iopub.status.idle":"2024-11-01T02:31:41.275847Z","shell.execute_reply.started":"2024-11-01T02:31:41.264154Z","shell.execute_reply":"2024-11-01T02:31:41.274955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"数据加载：文件解压、训练测试集的划分、图片与处理（裁剪、归一化和数据增强）","metadata":{}},{"cell_type":"code","source":"# 在kaggle上解压数据\ntrain_path = \"/kaggle/input/dogs-vs-cats/train.zip\"\ntest_path = \"/kaggle/input/dogs-vs-cats/test1.zip\"#训练集和测试集的路径\n\nfiles = \"/kaggle/working/\"\n\nimport zipfile#用zipfile来解压放到file中\n\nwith zipfile.ZipFile(train_path, 'r') as zipp:\n    zipp.extractall(files)\n    \nwith zipfile.ZipFile(test_path, 'r') as zipp:\n    zipp.extractall(files)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:31:41.277059Z","iopub.execute_input":"2024-11-01T02:31:41.277442Z","iopub.status.idle":"2024-11-01T02:31:51.139471Z","shell.execute_reply.started":"2024-11-01T02:31:41.277397Z","shell.execute_reply":"2024-11-01T02:31:51.138448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip show torchvision\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:31:51.142433Z","iopub.execute_input":"2024-11-01T02:31:51.143165Z","iopub.status.idle":"2024-11-01T02:32:02.233958Z","shell.execute_reply.started":"2024-11-01T02:31:51.143115Z","shell.execute_reply":"2024-11-01T02:32:02.232717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchvision pillow","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:32:02.235465Z","iopub.execute_input":"2024-11-01T02:32:02.235786Z","iopub.status.idle":"2024-11-01T02:32:13.945829Z","shell.execute_reply.started":"2024-11-01T02:32:02.235751Z","shell.execute_reply":"2024-11-01T02:32:13.944509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mydata(Dataset):\n    \"\"\"定义自己的数据集\"\"\"\n    def __init__(self, root, Transforms=None, train=True):\n        \"\"\"\n        Args:\n            root: 训练集的路径\n            Transforms: 图片处理的方式\n            train: 是否是训练数据。True: 训练数据; False: 验证数据\n        \"\"\"\n        imgs = [os.path.join(root, img) for img in os.listdir(root)]\n        # 按序号排序，以便训练集和验证集不会混乱\n        # 假设文件名格式为 '类别_编号.jpg'，例如 'dog_6809.jpg'\n        def get_sort_key(filename):\n            filename = filename.split(\"/\")[-1]\n            parts = filename.split(\".\")\n            if len(parts) > 1 and parts[0] in ['dog', 'cat']:\n                return int(parts[1].split('.')[0])\n            return 0  # 如果不符合预期格式，返回一个默认值\n\n        imgs = sorted(imgs, key=get_sort_key)\n        imgs_num = len(imgs)\n        \"\"\"进行数据集的划分\"\"\"\n        if train:\n            self.imgs = imgs[:int(0.8 * imgs_num)]  # 80% 训练集\n        else:\n            self.imgs = imgs[int(0.8 * imgs_num):]  # 20% 验证集\n        if Transforms is None:\n            normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                                            std=[0.229, 0.224, 0.225])\n            # 图片处理\n            color_aug = transforms.ColorJitter(brightness=0.1)\n            self.transforms = transforms.Compose(\n                [transforms.Resize([224, 224]),              #图片剪裁\n                 transforms.RandomHorizontalFlip(p=0.5),    # 数据增强，包括随机水平旋转和随机垂直旋转，这是常用的图片数据增强的手段\n                 transforms.RandomVerticalFlip(p=0.5),\n                 transforms.ColorJitter(brightness=0.1),   # 亮度变化\n                 transforms.ToTensor(), #图片归一化，将 PIL 图像或 NumPy 数组转换为 PyTorch 张量，并将像素值归一化到 [0, 1] 范围内\n                 transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])\n\n    def __getitem__(self, index):\n        \"\"\"\n        返回一张图片的数据\n        训练集和验证集，则对应的是dog返回1,猫则返回0\n        \"\"\"\n        img_path = self.imgs[index]\n        label = 1 if 'dog' in img_path.split('/')[-1] else 0  # 获得标签\n        data = Image.open(img_path).convert(\"RGB\")  # 可能有单通道的，注意转换为三通道图片\n        try:\n            data = self.transforms(data)  # 图片处理\n        except:\n            print(img_path)\n            raise ValueError(\"图片打开失败\")\n        return data, label\n\n    def __len__(self):\n        \"\"\"返回数据集中所有图片的个数\"\"\"\n        return len(self.imgs)\n\n    def getall(self):\n        return self.imgs\n\nif __name__ == \"__main__\":\n    root = \"/kaggle/working/train\"\n    train_dataset = Mydata(root, train=True)\n    test_dataset = Mydata(root, train=False)\n    \n    img, label = train_dataset.__getitem__(5)\n    imgs = train_dataset.getall()\n    img = img.numpy()\n    print(type(img))\n    print(img.shape, label)\n    print(len(train_dataset))\n    print(len(test_dataset))\n    \n    plt.imshow(np.transpose(img, (1, 2, 0)))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:32:13.947659Z","iopub.execute_input":"2024-11-01T02:32:13.948087Z","iopub.status.idle":"2024-11-01T02:32:14.484817Z","shell.execute_reply.started":"2024-11-01T02:32:13.948036Z","shell.execute_reply":"2024-11-01T02:32:14.483615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AlexNet模型搭建","metadata":{}},{"cell_type":"code","source":"class AlexNet(nn.Module):\n    def __init__(self,classes=2):\n        super(AlexNet,self).__init__()\n        #第一个块,这里的padding=3 是由于 数据集使用的是224*224的图像，padding=3 \n        self.zeropadding2d=nn.ZeroPad2d(padding=(3,0,3,0))\n        self.conv1=nn.Conv2d(in_channels=3,out_channels=96,kernel_size=(11,11),stride=4) \n        self.bn1=nn.BatchNorm2d(96)  \n        self.relu1 = nn.ReLU()\n        self.maxpool1=nn.MaxPool2d(kernel_size=(3,3),stride=2)\n        \n        #第二个块\n        self.conv2=nn.Conv2d(in_channels=96,out_channels=256,kernel_size=(5,5),padding='same',stride=1)\n        self.bn2=nn.BatchNorm2d(256)\n        self.relu2 = nn.ReLU()\n        self.maxpool2=nn.MaxPool2d(kernel_size=(3,3),stride=2)\n        \n        #第三个快\n        self.conv3=nn.Conv2d(in_channels=256,out_channels=384,kernel_size=(5,5),padding='same',stride=1)\n        self.relu3 = nn.ReLU()\n        self.maxpool3=nn.MaxPool2d(kernel_size=(3,3),stride=2)\n        #第四~五块\n        self.conv4=nn.Conv2d(in_channels=384,out_channels=384,kernel_size=(3,3),padding='same',stride=1)\n        self.conv5=nn.Conv2d(in_channels=384,out_channels=256,kernel_size=(3,3),padding='same',stride=1)\n        self.relu4 = nn.ReLU()\n        self.maxpool4=nn.MaxPool2d(kernel_size=(3,3),stride=2)\n        #self.globalmax=nn.AdaptiveAvgPool2d()\n        self.flatten=nn.Flatten()\n        self.linear1=nn.Linear(in_features=1024,out_features=4096)\n        self.linear_relu1 = nn.ReLU()\n        self.linear2=nn.Linear(in_features=4096,out_features=4096)\n        self.linear_relu2 = nn.ReLU()\n        self.linear3=nn.Linear(in_features=4096,out_features=classes)\n        self.softmax=nn.Softmax(dim=1)\n    #定义计算过程\n    def forward(self,x):\n        x=self.zeropadding2d(x)\n        x=self.conv1(x)\n        x=self.bn1(x)\n        x=self.relu1(x)\n        x=self.maxpool1(x)\n\n        x=self.conv2(x)\n        x=self.bn2(x)\n        x=self.relu2(x)\n        x=self.maxpool2(x)\n\n        x=self.conv3(x)\n        x=self.relu3(x)\n        x=self.maxpool3(x)\n\n        x=self.conv4(x)\n        x=self.conv5(x)\n        x=self.relu4(x)\n        x=self.maxpool4(x)\n        x=self.flatten(x)\n\n        #linear connect\n        x=self.linear1(x)\n        x=self.linear_relu1(x)\n        x=self.linear2(x)\n        x=self.linear_relu2(x)\n        x=self.linear3(x)\n        #self.softmax(x)\n        return x\nif __name__==\"__main__\":\n    net=AlexNet()\n    \n    print(net,(3,224,224))#AlexNet模型实例","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:32:14.486617Z","iopub.execute_input":"2024-11-01T02:32:14.487320Z","iopub.status.idle":"2024-11-01T02:32:14.743456Z","shell.execute_reply.started":"2024-11-01T02:32:14.487270Z","shell.execute_reply":"2024-11-01T02:32:14.742452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"模型训练","metadata":{}},{"cell_type":"code","source":"\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import StepLR\ndef train(root, epoch_num, model_name, batch_size, lr_decay, resume=False, pre=False):\n    # 加载数据\n    trainset = Mydata(root, train=True)\n    validset = Mydata(root, train=False)\n    train_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=3)\n    test_loader = DataLoader(validset, batch_size=batch_size, shuffle=False, num_workers=3)\n\n    # 加载模型和超参\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model = AlexNet(classes=2)  # 定义模型\n    model = model.to(device)\n\n    # 初始化损失函数和优化器\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.0001)\n    scheduler = StepLR(optimizer, step_size=5, gamma=0.1)\n\n    # 保存模型的文件夹\n    save_foldername = \"saved_models/\" + model_name\n    os.makedirs(save_foldername, exist_ok=True)\n\n    writer = SummaryWriter(save_foldername)\n\n    # 开始训练\n    print('开始训练...')\n    best_accuracy = 0\n\n    for epoch in range(epoch_num):\n        model.train()\n        train_loss = 0\n        correct = 0\n        total = 0\n\n        for data, target in tqdm(train_loader):\n            data, target = data.to(device), target.to(device)\n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n            train_loss += loss.item()\n            loss.backward()\n            optimizer.step()\n             #量化计算分析\n            _, predicted = torch.max(output.data, 1)\n            total += target.size(0)\n            correct += (predicted == target).sum().item()\n\n        # 训练过程评估\n        train_accuracy = correct / total\n        Loss, accuracy = validation(model, test_loader, device, criterion)\n\n        # 打印训练和验证损失及准确率\n        print(f'Epoch [{epoch + 1}/{epoch_num}], '\n              f'Train Loss: {train_loss / len(train_loader):.4f}, '\n              f'Train Acc: {train_accuracy:.2f}, '\n              f'Val Loss: {Loss:.4f}, '\n              f'Val Acc: {accuracy:.2f}')\n        # 监控训练和验证\n        writer.add_scalar('Train Loss', train_loss / len(train_loader), epoch)\n        writer.add_scalar('Train Accuracy', train_accuracy, epoch)\n        writer.add_scalar('Val Loss', Loss, epoch)\n        writer.add_scalar('Val Accuracy', accuracy, epoch)\n        # 保存最佳模型\n        if accuracy > best_accuracy:\n            best_accuracy = accuracy\n            torch.save(model.state_dict(), save_foldername + \"/\" + model_name + \"_best.pth\")\n\n    writer.close()\n\n@torch.no_grad()\ndef validation(model, test_loader, device, criterion):\n    model.eval()\n    total = 0\n    correct = 0\n    test_loss = 0\n    \n    for data, labels in test_loader:\n        images, labels = data.to(device), labels.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        loss = criterion(outputs, labels)\n        test_loss += loss.item()\n\n    accuracy = 100 * correct / total\n    Loss = test_loss / total if total > 0 else float('inf')\n    return Loss, accuracy\n\nif __name__ == \"__main__\":\n    root = \"/kaggle/working/train\"  # 训练数据集路径\n    epoch_num = 20  # 迭代次数\n    modelname = \"AlexNet\"  # 模型名字\n    batchsize = 30  # batchsize\n    resume = False  # 是否接着上次的继续训练\n    lr_decay = 0.9  # 学习率下降率\n    pre = False  # 是否采用预训练模型\n\n    start = time.time()\n    train(root, epoch_num, modelname, batchsize, lr_decay, resume, pre)\n    end = time.time()\n    print(f\"训练耗时：{round((end - start) / 60, 2)}分钟\")","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:32:14.744738Z","iopub.execute_input":"2024-11-01T02:32:14.745030Z","iopub.status.idle":"2024-11-01T02:50:51.722631Z","shell.execute_reply.started":"2024-11-01T02:32:14.744999Z","shell.execute_reply":"2024-11-01T02:50:51.721371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"进行预测","metadata":{}},{"cell_type":"code","source":"def predict(root, imgname, model, img_trans):\n    #加载和预处理图像\n    imgpath = os.path.join(root, imgname)\n    img_rgb = Image.open(imgpath).convert(\"RGB\")\n    img_tensor = img_trans(img_rgb).unsqueeze(0)  # 增加一个维度，表示批量大小\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model = model.to(device)\n    img_tensor = img_tensor.to(device)\n  #模型预测\n    with torch.no_grad():  # 不计算梯度\n        output = model(img_tensor)\n        prob = torch.softmax(output, dim=1)\n        pred_prob, pred_idx = torch.max(prob, dim=1)\n        pred_prob = pred_prob.item()\n        pred_idx = pred_idx.item()\n\n    # 将索引转换为标签\n    pred_str = \"cat\" if pred_idx == 0 else \"dog\"\n\n    # 显示并保存结果\n    plt.imshow(img_rgb)\n    plt.title(\"Predicted: {} , Probability: {:.2f}\".format(pred_str, pred_prob))\n    plt.axis('off')  # 不显示坐标轴\n    plt.savefig(\"output/pre_\" + imgname)  # 保存预测结果的图像\n\nif __name__ == \"__main__\":\n    # 图像预处理\n    img_trans = transforms.Compose([\n        transforms.Resize([224, 224]),  # 先调整尺寸\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n     #加载模型\n    model = AlexNet() \n    modelpath = \"/kaggle/working/saved_models/AlexNet/AlexNet_best.pth\"  # 训练好的模型路径\n    checkpoint = torch.load(modelpath, weights_only=True)\n    model_dict = model.state_dict()\n\n     #  加载匹配的参数，并将模型设置为评估模式\n    filtered_checkpoint = {k: v for k, v in checkpoint.items() if k in model_dict}\n    model_dict.update(filtered_checkpoint) \n    model.load_state_dict(model_dict)\n    model.eval()\n    #处理测试集，指定测试集路径、确保输出文件存在，并对每张图片进行预测\n    root = \"/kaggle/working/test1\"\n    os.makedirs(\"output\", exist_ok=True)  # 确保输出目录存在\n    for pic in os.listdir(root):\n        if pic.endswith(\".jpg\"):\n            predict(root, pic, model, img_trans)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T02:50:51.724822Z","iopub.execute_input":"2024-11-01T02:50:51.725569Z"},"trusted":true},"execution_count":null,"outputs":[]}]}