{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"先调包","metadata":{}},{"cell_type":"code","source":"!pip install d2l\n!pip install py7zr\n\n\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom d2l import torch as d2l\nfrom torchvision.transforms import ToTensor\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.datasets import ImageFolder\nimport d2l\nfrom torch import nn\nimport pandas as pd\nfrom PIL import Image\nimport torchvision.transforms as transforms\nimport torchvision\nimport os\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport csv\nimport torchvision.models as models\nimport py7zr\nfrom PIL import Image, ImageFilter","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:39:28.014268Z","iopub.execute_input":"2023-05-25T14:39:28.014838Z","iopub.status.idle":"2023-05-25T14:48:34.976451Z","shell.execute_reply.started":"2023-05-25T14:39:28.014807Z","shell.execute_reply":"2023-05-25T14:48:34.975381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"使用模型resnet18，解压文件(文件单独解压再上传了)","metadata":{}},{"cell_type":"code","source":"net = models.resnet18(pretrained=True)\n\n#def extract_7z(file_path, output_path):\n#    with py7zr.SevenZipFile(file_path, mode='r') as z:\n#        z.extractall(path=output_path)\n\n#extract_7z('/kaggle/input/cifar-10/train.7z', '/kaggle/working/')\n#extract_7z('/kaggle/input/cifar-10/test.7z', '/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:34.978832Z","iopub.execute_input":"2023-05-25T14:48:34.979226Z","iopub.status.idle":"2023-05-25T14:48:35.535389Z","shell.execute_reply.started":"2023-05-25T14:48:34.979184Z","shell.execute_reply":"2023-05-25T14:48:35.534285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"自定义锐化图片的函数，提高特征","metadata":{}},{"cell_type":"code","source":"# 定义锐化的自定义转换函数\nclass Sharpen(object):\n    def __call__(self, image):\n        # 转换为PIL图像对象\n        image_pil = transforms.ToPILImage()(image)\n\n        # 应用锐化滤波器\n        sharpened_image_pil = image_pil.filter(ImageFilter.SHARPEN)\n\n        # 转换回Tensor对象\n        sharpened_image = transforms.ToTensor()(sharpened_image_pil)\n\n        return sharpened_image","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:35.536972Z","iopub.execute_input":"2023-05-25T14:48:35.537356Z","iopub.status.idle":"2023-05-25T14:48:35.544165Z","shell.execute_reply.started":"2023-05-25T14:48:35.537322Z","shell.execute_reply":"2023-05-25T14:48:35.543095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"使标签和序号对应","metadata":{}},{"cell_type":"code","source":"labels_dataframe = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')\nleaves_labels = sorted(list(set(labels_dataframe['label'])))\nn_classes = len(leaves_labels)\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nnum_to_class = {v : k for k, v in class_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:35.547135Z","iopub.execute_input":"2023-05-25T14:48:35.547482Z","iopub.status.idle":"2023-05-25T14:48:35.58866Z","shell.execute_reply.started":"2023-05-25T14:48:35.547448Z","shell.execute_reply":"2023-05-25T14:48:35.587533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"对csv图片预处理，并且使用k作为折数，0.2是取的数据集一部分用于测试，并且添加了一些参数，并在kaggle上训练","metadata":{}},{"cell_type":"code","source":"# 继承pytorch的dataset，创建自己的\nclass LeavesData(Dataset):\n    def __init__(self, csv_path, file_path, k=0, mode='train', valid_ratio=0.2, resize_height=256, resize_width=256):\n        \"\"\"\n        Args:\n            csv_path (string): csv 文件路径\n            img_path (string): 图像文件所在路径\n            mode (string): 训练模式还是测试模式\n            valid_ratio (float): 验证集比例\n        \"\"\"\n        \n        # 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#\n        self.resize_height = resize_height\n        self.resize_width = resize_width\n\n        self.file_path = file_path\n        self.k=k\n        self.mode = mode\n\n        # 读取 csv 文件\n        # 利用pandas读取csv文件\n        self.data_info = pd.read_csv(csv_path, header=None)  #header=None是去掉表头部分\n        # 计算 length\n        self.data_len = len(self.data_info.index) - 1\n        self.train_len = int(self.data_len * valid_ratio)-1\n        \n        \n        if mode == 'train':\n            # 第一列包含图像文件的名称,self.data_info.iloc[1:,0]表示读取第一列，从第二行开始到train_len\n            self.train_image = np.asarray(pd.concat([self.data_info.iloc[1:self.k*self.train_len, 0],\n                                                     self.data_info.iloc[1+(self.k+1)*self.train_len:, 0]]))  \n            # 第二列是图像的 label\n            self.train_label = np.asarray(pd.concat([self.data_info.iloc[1:self.k*self.train_len, 1],\n                                                     self.data_info.iloc[1+(self.k+1)*self.train_len:, 1]]))  \n            self.image_arr = self.train_image \n            self.label_arr = self.train_label\n        elif mode == 'valid':\n            self.valid_image = np.asarray(self.data_info.iloc[1+self.k*self.train_len:1+(self.k+1)*self.train_len, 0])  \n            self.valid_label = np.asarray(self.data_info.iloc[1+self.k*self.train_len:1+(self.k+1)*self.train_len, 1])\n            self.image_arr = self.valid_image\n            self.label_arr = self.valid_label\n        elif mode == 'test':\n            self.test_image = np.asarray(self.data_info.iloc[1:, 0])\n            self.image_arr = self.test_image\n            \n        self.real_len = len(self.image_arr)\n\n        print('Finished reading the {} set of Leaves Dataset ({} samples found)'\n              .format(mode, self.real_len))\n\n    def __getitem__(self, index):\n        # 从 image_arr中得到索引对应的文件名\n        single_image_name = self.image_arr[index]\n\n        # 读取图像文件\n        img_as_img = Image.open(self.file_path + single_image_name+'.png')\n\n        #如果需要将RGB三通道的图片转换成灰度图片可参考下面两行\n#         if img_as_img.mode != 'L':\n#             img_as_img = img_as_img.convert('L')\n\n        #设置好需要转换的变量，还可以包括一系列的nomarlize等等操作\n        if self.mode == 'train':\n            transform = transforms.Compose([\n                torchvision.transforms.Resize(40),\n                # 随机裁剪出一个高度和宽度均为40像素的正方形图像\n                # 生成一个面积为原始图像面积0.64～1倍的小正方形\n                # 然后将其缩放为高度和宽度均为32像素的正方形\n                torchvision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0),ratio=(1.0, 1.0)),\n                transforms.RandomHorizontalFlip(p=0.5),\n                transforms.ToTensor(),\n                Sharpen(),  # 自定义的锐化预处理操作\n                transforms.Resize((224, 224)),\n                torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])\n            ])\n        else:\n            # valid和test不做数据增强,但是也进行锐化\n            transform = transforms.Compose([\n                transforms.ToTensor(),\n                Sharpen(),  # 自定义的锐化预处理操作\n                transforms.Resize((224, 224)),\n                torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])\n            ])\n        \n        img_as_img = transform(img_as_img)\n        \n        if self.mode == 'test':\n            return img_as_img\n        else:\n            # 得到图像的 string label\n            label = self.label_arr[index]\n            # number label\n            number_label = class_to_num[label]\n\n            return img_as_img, number_label  #返回每一个index对应的图片数据和对应的label\n\n    def __len__(self):\n        return self.real_len\n    \nnum_epochs = 15\nbatch_size=16\nlr=0.05\n    \nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nos.environ['CUDA_LAUNCH_BLOCKING'] = '1'\n\nif torch.cuda.is_available():\n    torch.cuda.init()\n    print('GPU available')\n    \ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nnet.to(device)  # 将模型移动到设备上（如GPU）\n\ntrain_path = '/kaggle/input/cifar-10/trainLabels.csv'\n# csv文件中已经images的路径了，因此这里只到上一级目录\nimg_path = '/kaggle/input/cifar-10-unzip/train/'","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:35.590381Z","iopub.execute_input":"2023-05-25T14:48:35.590739Z","iopub.status.idle":"2023-05-25T14:48:38.790619Z","shell.execute_reply.started":"2023-05-25T14:48:35.590704Z","shell.execute_reply":"2023-05-25T14:48:38.7896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"定义训练函数，i是循环使用折数k折交叉验证，目前训15轮","metadata":{}},{"cell_type":"code","source":"def train_ch_def(net, num_epochs, lr, device):\n    loss = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.SGD(net.parameters(), lr=lr)\n    net = net.to(device)\n    \n    for i in range(5):\n        train_dataset = LeavesData(train_path, img_path,k=i,mode='train')\n        val_dataset = LeavesData(train_path, img_path, k=i,mode='valid')\n\n        train_iter = DataLoader(dataset=train_dataset,batch_size=batch_size, shuffle=True)#表示随机洗牌读数\n        val_iter = DataLoader(dataset=val_dataset,batch_size=batch_size, shuffle=True)\n        \n        for epoch in range(num_epochs):\n            net.train()\n            train_loss, train_acc, n = 0.0, 0.0, 0\n            for X, y in train_iter:\n                X = X.to(device)\n                y = y.to(device)\n                optimizer.zero_grad()\n                y_hat = net(X)\n                l = loss(y_hat, y)\n                l.backward()\n                optimizer.step()\n                train_loss += l.item() * y.shape[0]\n                train_acc += (y_hat.argmax(dim=1) == y).sum().item()\n                n += y.shape[0]\n\n            net.eval()\n            val_loss, val_acc, m = 0.0, 0.0, 0\n            for X, y in val_iter:\n                X = X.to(device)\n                y = y.to(device)\n                with torch.no_grad():\n                    y_hat = net(X)\n                    l = loss(y_hat, y)\n                    val_loss += l.item() * y.shape[0]\n                    val_acc += (y_hat.argmax(dim=1) == y).sum().item()\n                    m += y.shape[0]\n\n            train_loss /= n\n            train_acc /= n\n            val_loss /= m\n            val_acc /= m\n\n            print(f\"Epoch {i+1}.{epoch+1}: Train Loss={train_loss:.4f}, Train Acc={train_acc:.4f}, Val Loss={val_loss:.4f}, Val Acc={val_acc:.4f}\")\n        break","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:38.792134Z","iopub.execute_input":"2023-05-25T14:48:38.792686Z","iopub.status.idle":"2023-05-25T14:48:38.805478Z","shell.execute_reply.started":"2023-05-25T14:48:38.79265Z","shell.execute_reply":"2023-05-25T14:48:38.804505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"训练完成后输出模型","metadata":{}},{"cell_type":"code","source":"train_ch_def(net, num_epochs, lr, device)\ntorch.save(net.state_dict(), '/kaggle/working/cifar10-resnet18.pth')","metadata":{"execution":{"iopub.status.busy":"2023-05-25T14:48:38.807108Z","iopub.execute_input":"2023-05-25T14:48:38.807503Z","iopub.status.idle":"2023-05-25T15:35:57.119767Z","shell.execute_reply.started":"2023-05-25T14:48:38.80747Z","shell.execute_reply":"2023-05-25T15:35:57.1181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"复制输出文件到working文件夹里","metadata":{}},{"cell_type":"code","source":"import shutil\n\n# 定义源文件路径和目标路径\nsource_file = '/kaggle/input/cifar-10/sampleSubmission.csv'\ndestination_folder = '/kaggle/working/sampleSubmission.csv'\n\n# 使用shutil.copy()函数复制文件\nshutil.copy(source_file, destination_folder)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:35:57.121145Z","iopub.status.idle":"2023-05-25T15:35:57.121861Z","shell.execute_reply.started":"2023-05-25T15:35:57.121608Z","shell.execute_reply":"2023-05-25T15:35:57.121632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"加载模型进行测试，并且把文件输出到csv提交文件里","metadata":{}},{"cell_type":"code","source":"net.load_state_dict(torch.load('/kaggle/working/cifar-resnet18.pth'))\nnet.eval()  # 设置模型为评估模式\n\ntest_path = '/kaggle/input/cifar-10/sampleSubmission.csv'\n# csv文件中已经images的路径了，因此这里只到上一级目录\nimg_path = '/kaggle/input/cifar-10-unzip/test/'\n\ntest_dataset = LeavesData(test_path, img_path, k=0,mode='test')\ntest_iter = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# 将模型移动到设备\nnet.to(device)\n\nwith open('sampleSubmission', 'w', newline='') as file:\n    writer = csv.writer(file)\n    writer.writerows(data)\n\ncsv_file = '/kaggle/working/sampleSubmission.csv'\nnum=0\n\nwith open(csv_file, 'w', newline='') as file:\n    writer = csv.writer(file)\n    writer.writerow(['id', 'label'])  # 写入表头\n\nfor images in test_iter:\n    # 对图像进行预处理\n    images = images.to(device)\n    outputs = model(images)\n    # 获取预测结果\n    _, predicted = torch.max(outputs, 1)\n\n    # 打印预测结果\n    #print(num_to_class[predicted])\n    class_predictions = [num_to_class[i.item()] for i in predicted]\n    \n    # 将列表元素按行写入CSV文件\n    with open(csv_file, 'a', newline='') as file:\n        writer = csv.writer(file)\n        if file.tell() == 0:  # 检查文件是否为空\n            writer.writerow(['id', 'label'])  # 写入表头\n        for index,element in enumerate(class_predictions):\n            writer.writerow([f'{1+index+num}', element])\n    num+=batch_size\n\n\nprint(\"列表元素已写入CSV文件：\", csv_file)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:35:57.12324Z","iopub.status.idle":"2023-05-25T15:35:57.12392Z","shell.execute_reply.started":"2023-05-25T15:35:57.123677Z","shell.execute_reply":"2023-05-25T15:35:57.123699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"查看预测结果","metadata":{}},{"cell_type":"code","source":"predict=pd.read_csv('/kaggle/working/sample_submission.csv')\npredict","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:35:57.125184Z","iopub.status.idle":"2023-05-25T15:35:57.125859Z","shell.execute_reply.started":"2023-05-25T15:35:57.125618Z","shell.execute_reply":"2023-05-25T15:35:57.12564Z"},"trusted":true},"execution_count":null,"outputs":[]}]}