{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nfrom torch import nn\nfrom torchvision import transforms, models, utils\nfrom torch.utils.data import Dataset,DataLoader,random_split\nfrom torch.utils.tensorboard import SummaryWriter \n\n# 路径\ninput_path = r'/kaggle/input/dogs-vs-cats-redux-kernels-edition/'\nroot_path = r'/kaggle/working/'\ntrain_path = os.path.join(root_path,'train')\ntest_path = os.path.join(root_path,'test')\ncsv_path = os.path.join(root_path,'test.csv')\ntsrbd_path = os.path.join(root_path,'tsrbd')\nmod_path = os.path.join(root_path,'model')\n\n# 设备检查\ndevice = (\n    \"cuda\" if torch.cuda.is_available()\n    else \"mps\" if torch.backends.mps.is_available()\n    else \"cpu\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:24:31.245659Z","iopub.execute_input":"2023-05-04T05:24:31.246006Z","iopub.status.idle":"2023-05-04T05:24:31.256412Z","shell.execute_reply.started":"2023-05-04T05:24:31.245978Z","shell.execute_reply":"2023-05-04T05:24:31.255424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatDogDataSet(Dataset):\n    \"\"\"\n    生成标准预处理图像数据集\n    \n    :inherits DataSet:\n    :param data_path: 数据路径\n    :param train: 是否为训练集\n    :param transform: 图像预处理方式\n    \"\"\"\n    \n    def __init__(self, data_path:str, train=True, transform=None):\n        self.data_path = data_path\n        self.train_flag = train\n        self.path_list = os.listdir(self.data_path)\n        \n        if transform is None:\n            # 设置图像默认预处理方式：调整尺寸为 300x300、转为张量并进行归一化、标准化处理\n            # 标准化：(x - 均值)/标准差。此处是为了使取值范围转为(-1,1)             \n            self.transform = transforms.Compose(\n            [\n                transforms.Resize((300,300)),\n                transforms.ToTensor(),\n                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n            ])\n        else:\n            self.transform = transform\n            \n    def __getitem__(self, idx: int):\n        \"\"\"\n        根据索引返回预处理后的图像与标记，如果是测试集，则将图片序号作为标记\n\n        :param idx: 索引\n        :return img: 图像\n        :return tag: 标记\n        \"\"\"\n        img_path = self.path_list[idx]\n        \n        label = img_path.split('.')[0]\n        label = (\n            int(label) if self.train_flag is False \n            else 1 if label == 'dog' else 0\n        )\n        \n        label = torch.as_tensor(label, dtype=torch.int64) \n        img = self.transform(Image.open(os.path.join(self.data_path, img_path)))\n        \n        return img, label\n    \n    def __len__(self) -> int:\n        return len(self.path_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:29.478809Z","iopub.execute_input":"2023-05-04T05:06:29.479613Z","iopub.status.idle":"2023-05-04T05:06:29.489541Z","shell.execute_reply.started":"2023-05-04T05:06:29.479575Z","shell.execute_reply":"2023-05-04T05:06:29.488576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取训练集与无标签测试集\ntrain_ds = CatDogDataSet(train_path)\ntest_ds = CatDogDataSet(test_path,train=False)\n\n# 将数据集按比例随机划分为训练集与有标签测试集，以检验准确率\nsub_train_size = int(0.8 * len(train_ds))\nsub_test_size = len(train_ds) - sub_train_size\n\nsub_train_ds, sub_test_ds = random_split(\n    train_ds,[sub_train_size, sub_test_size]\n)\n\n# 定义数据加载器，用于分批加载数据集\n# 使用 enumerate 迭代获取批次\ntrain_loader = DataLoader(\n    train_ds, batch_size=32, pin_memory=True, shuffle=True\n)\ntest_loader = DataLoader(\n    test_ds, batch_size=32, pin_memory=True, shuffle=True\n)\nsub_train_loader  = DataLoader(\n    sub_train_ds, batch_size=32, pin_memory=True, shuffle=True\n)\nsub_test_loader = DataLoader(\n    sub_train_ds, batch_size=32, pin_memory=True, shuffle=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:32.352735Z","iopub.execute_input":"2023-05-04T05:06:32.353092Z","iopub.status.idle":"2023-05-04T05:06:32.403467Z","shell.execute_reply.started":"2023-05-04T05:06:32.353064Z","shell.execute_reply":"2023-05-04T05:06:32.402502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 测试输出图片\nrandom_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure()\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = transforms.ToPILImage()(train_ds[idx][0]).convert('RGB')\n    plt.imshow(img)\n    i+=1\n\nplt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 使用 ResNet50 深度残差网络 作为预训练模型，默认权重\nmodel = models.resnet50(weights='DEFAULT')\n\n# 将 ResNet50 连接层的输出维度（分类）由 2048 改为 2\nmodel.fc = nn.Linear(2048,2)\n\n# 选择损失函数（交叉熵）和优化函数（随机梯度下降）\nloss = nn.CrossEntropyLoss()\nopt = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:37.840382Z","iopub.execute_input":"2023-05-04T05:06:37.840715Z","iopub.status.idle":"2023-05-04T05:06:38.992346Z","shell.execute_reply.started":"2023-05-04T05:06:37.840688Z","shell.execute_reply":"2023-05-04T05:06:38.991402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, device, loader, lossfunc, optizer, epoch):\n    \"\"\"\n    训练模型\n    \n    param: model: 模型\n    param: device: 设备标识\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    param: optizer: 优化函数\n    param: epoch: 迭代次数\n    \"\"\"\n    # 生成并训练模型   \n    model = model.to(device)\n    # 转换成 tqdm 类型，便于输出\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    for e in range(epoch):\n        losses = 0.0\n        model.train()\n        loader.set_description(f'迭代次数 [{e+1}/{epoch}]')\n        for i, (images, labels) in loader:\n            # 重置下降梯度\n            opt.zero_grad()\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 误差\n            loss = lossfunc(outputs,labels)\n            losses += loss.item()\n            # 误差逆传播\n            loss.backward()\n            # 迭代优化\n            opt.step()\n            \n    print('训练结束')","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:08:46.557345Z","iopub.execute_input":"2023-05-04T06:08:46.558029Z","iopub.status.idle":"2023-05-04T06:08:46.565221Z","shell.execute_reply.started":"2023-05-04T06:08:46.557997Z","shell.execute_reply":"2023-05-04T06:08:46.564302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(model, device, loader, lossfunc):\n    \"\"\"\n    测试模型\n    \n    param: model: 模型\n    param: device: 设备\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    \"\"\"\n    losses = 0.0\n    correct = 0\n    total = 0\n    \n    # 生成并测试模型\n    model = model.to(device)\n    model.eval()\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    \n    # 关闭梯度计算\n    with torch.no_grad():\n        for i, (images, labels) in loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 误差\n            loss = lossfunc(outputs,labels)\n            losses += loss.item()\n            # 预测结果\n            _,pred = torch.max(outputs.data,1)\n            # 准确率\n            total += labels.size(0)\n            correct += (pred == labels).sum().item()\n            \n            loader.set_postfix(loss=losses/total, acc=correct/total)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:54.623245Z","iopub.execute_input":"2023-05-04T05:06:54.623835Z","iopub.status.idle":"2023-05-04T05:06:54.630292Z","shell.execute_reply.started":"2023-05-04T05:06:54.623803Z","shell.execute_reply":"2023-05-04T05:06:54.629422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(model, device, loader, lossfunc, csv_path):\n    \"\"\"\n    测试模型并输出预测结果\n    \n    param: model: 模型\n    param: device: 设备\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    param: csv_path: 输出路径\n    \"\"\"\n    rs = []\n    \n    # 生成并测试模型\n    model = model.to(device)\n    model.eval()\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    \n    # 关闭梯度计算\n    with torch.no_grad():\n        for i, (images, labels) in loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 预测结果\n            _,pred = nn.Softmax(1)(outputs).max(dim=1)\n            # 写入列表\n            for p in range(len(pred)):\n                rs.append({\n                    \"id\": labels[p].item(),\n                    \"label\": pred[p].item()\n                })\n    # 转成 dataframe 并保存为 csv\n    pd.DataFrame(rs,columns=rs[0].keys()).sort_values(\"id\").to_csv(csv_path, index=None)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:41:21.159473Z","iopub.execute_input":"2023-05-04T05:41:21.162542Z","iopub.status.idle":"2023-05-04T05:41:21.174882Z","shell.execute_reply.started":"2023-05-04T05:41:21.162499Z","shell.execute_reply":"2023-05-04T05:41:21.172259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(model,device,sub_train_loader,loss,opt,60)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:08:50.750536Z","iopub.execute_input":"2023-05-04T06:08:50.750879Z","iopub.status.idle":"2023-05-04T06:12:04.618273Z","shell.execute_reply.started":"2023-05-04T06:08:50.750852Z","shell.execute_reply":"2023-05-04T06:12:04.617236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(model,device,sub_test_loader,loss)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(csv_path):\n    os.remove(csv_path)\nsubmit(model, device, test_loader, loss, csv_path)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:02:17.059041Z","iopub.execute_input":"2023-05-04T06:02:17.059471Z","iopub.status.idle":"2023-05-04T06:04:00.647254Z","shell.execute_reply.started":"2023-05-04T06:02:17.059437Z","shell.execute_reply":"2023-05-04T06:04:00.646267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(root_path,'test.csv')).sample(n=10)\n\nfig, axs = plt.subplots(nrows=2, ncols=5, figsize=(12, 6),\n                        subplot_kw={'xticks': [], 'yticks': []})\n\nfor i, ax in enumerate(axs.flat):\n    img = df.iloc[i, 0]\n    label = df.iloc[i, 1]\n\n    # 加载图像并将其转换为numpy数组\n    img = Image.open(os.path.join(test_path,f'{img}.jpg'))\n\n    # 显示图像并添加标签\n    ax.imshow(np.array(img))\n    ax.set_title('dog' if label == 0 else 'cat')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:04:05.256343Z","iopub.execute_input":"2023-05-04T06:04:05.256702Z","iopub.status.idle":"2023-05-04T06:04:06.257834Z","shell.execute_reply.started":"2023-05-04T06:04:05.256674Z","shell.execute_reply":"2023-05-04T06:04:06.255967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}