{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd#导入csv文件的库\nimport numpy as np#进行矩阵运算的库\nimport matplotlib.pyplot as plt#作图的库\nimport torch#一个深度学习的库Pytorch\nimport torch.nn as nn#neural network,神经网络\nfrom torch.autograd import Variable#从自动求导中引入变量\nimport torch.optim as optim#一个实现了各种优化算法的库\nimport torch.nn.functional as F#神经网络函数库\nimport os#与操作系统交互,处理文件和目录、管理进程、获取环境变量\nfrom PIL import Image, ImageOps, ImageFilter, ImageEnhance#PIL是图像处理库\nimport torchvision.transforms as transforms#图像、视频、文本增强和预处理的库\nimport zipfile#对ZIP格式的归档文件进行读取、写入和操作\nimport warnings#避免一些可以忽略的报错\nwarnings.filterwarnings('ignore')#filterwarnings()方法是用于设置警告过滤器的方法，它可以控制警告信息的输出方式和级别。","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:57.268999Z","iopub.execute_input":"2023-04-15T07:21:57.269388Z","iopub.status.idle":"2023-04-15T07:21:59.722646Z","shell.execute_reply.started":"2023-04-15T07:21:57.269354Z","shell.execute_reply":"2023-04-15T07:21:59.721063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/dogs-vs-cats-redux-kernels-edition')#返回指定路径下的文件和目录名的列表","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:59.724437Z","iopub.execute_input":"2023-04-15T07:21:59.724825Z","iopub.status.idle":"2023-04-15T07:21:59.734836Z","shell.execute_reply.started":"2023-04-15T07:21:59.724801Z","shell.execute_reply":"2023-04-15T07:21:59.733619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_files = ['test', 'train']\n\nfor zip_file in zip_files:\n    with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}.zip\".format(zip_file),\"r\") as z:\n        z.extractall(\".\")#解压\n        print(\"{} unzipped\".format(zip_file))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:59.736310Z","iopub.execute_input":"2023-04-15T07:21:59.736730Z","iopub.status.idle":"2023-04-15T07:22:18.547696Z","shell.execute_reply.started":"2023-04-15T07:21:59.736695Z","shell.execute_reply":"2023-04-15T07:22:18.546567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_Folder_Path = \"../working/train\"\nTest_Folder_Path = \"../working/test\"\ntrain_file_names = os.listdir(Train_Folder_Path)\ntest_file_names = os.listdir(Test_Folder_Path)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.550374Z","iopub.execute_input":"2023-04-15T07:22:18.550694Z","iopub.status.idle":"2023-04-15T07:22:18.571570Z","shell.execute_reply.started":"2023-04-15T07:22:18.550666Z","shell.execute_reply":"2023-04-15T07:22:18.570111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\nfull_paths = []\nfor file_name in train_file_names:\n    target = file_name.split(\".\")[0]\n    full_path = os.path.join(Train_Folder_Path, file_name)\n    full_paths.append(full_path)\n    targets.append(target)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.572904Z","iopub.execute_input":"2023-04-15T07:22:18.573238Z","iopub.status.idle":"2023-04-15T07:22:18.631252Z","shell.execute_reply.started":"2023-04-15T07:22:18.573209Z","shell.execute_reply":"2023-04-15T07:22:18.629594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame()\ntrain_df['image_path'] = full_paths\ntrain_df['target'] = targets","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.633114Z","iopub.execute_input":"2023-04-15T07:22:18.633444Z","iopub.status.idle":"2023-04-15T07:22:18.658013Z","shell.execute_reply.started":"2023-04-15T07:22:18.633412Z","shell.execute_reply":"2023-04-15T07:22:18.656803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_paths = []\nfor file_name in test_file_names:\n    full_path = os.path.join(Test_Folder_Path, file_name)\n    full_paths.append(full_path)\n\ntest_df = pd.DataFrame()\ntest_df['image_path'] = full_paths","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.659997Z","iopub.execute_input":"2023-04-15T07:22:18.660546Z","iopub.status.idle":"2023-04-15T07:22:18.689650Z","shell.execute_reply.started":"2023-04-15T07:22:18.660417Z","shell.execute_reply":"2023-04-15T07:22:18.688693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#定义正则化的标准，3个颜色通道的均值和方差,这是从ImageNet数据集上得出的数值\nnormalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.690898Z","iopub.execute_input":"2023-04-15T07:22:18.691882Z","iopub.status.idle":"2023-04-15T07:22:18.704111Z","shell.execute_reply.started":"2023-04-15T07:22:18.691851Z","shell.execute_reply":"2023-04-15T07:22:18.702638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    #transforms.RandomRotation(30), #在[-30,30]的范围内随机旋转一个角度\n    transforms.RandomResizedCrop(size=128, scale=(0.8, 1.2),interpolation=transforms.functional.InterpolationMode.BILINEAR),#先将图像随机进行缩放操作,然后再将图像变成128*128的图像\n    #transforms.RandomHorizontalFlip(), # 随机水平翻转\n    #transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5), # 随机颜色变换\n    transforms.ToTensor(),#转换成张量形式\n    normalize#标准化操作\n])","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.706747Z","iopub.execute_input":"2023-04-15T07:22:18.707109Z","iopub.status.idle":"2023-04-15T07:22:18.716996Z","shell.execute_reply.started":"2023-04-15T07:22:18.707083Z","shell.execute_reply":"2023-04-15T07:22:18.715873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgcnt=25000\ndata=[]\nlabel=[]\nfor i in range(imgcnt):\n    img=Image.open(train_df[\"image_path\"][i])#读取图片\n    new_img=transform(img)\n    data.append(new_img.detach().cpu().numpy())\n    if train_df['target'][i]==\"cat\":\n        label.append(0)#猫是0,狗是1\n    else:\n        label.append(1)\n    if i%500==0:\n        print(i)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:22:18.720300Z","iopub.execute_input":"2023-04-15T07:22:18.721197Z","iopub.status.idle":"2023-04-15T07:24:11.102567Z","shell.execute_reply.started":"2023-04-15T07:22:18.721161Z","shell.execute_reply":"2023-04-15T07:24:11.101592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#划分训练集和测试集\ndata=np.array(data)\nlabel=np.array(label)\nnum=[i for i in range(imgcnt)]\nnp.random.shuffle(num)\ndata=data[num]\nlabel=label[num]\ntrain_X=data[:8*imgcnt//10]\ntrain_y=label[:8*imgcnt//10]\ntest_X=data[8*imgcnt//10:]\ntest_y=label[8*imgcnt//10:]","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:24:11.104100Z","iopub.execute_input":"2023-04-15T07:24:11.105072Z","iopub.status.idle":"2023-04-15T07:24:12.627522Z","shell.execute_reply.started":"2023-04-15T07:24:11.105032Z","shell.execute_reply":"2023-04-15T07:24:12.626065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n\n    def __init__(self):\n        \n        super(CNN,self).__init__()\n       \n        self.model=nn.Sequential()\n        \n        #3*128*128\n        self.model.add_module('conv1_1',nn.Conv2d(3,32,3,1,0,bias=False))\n        #32*126*126\n        self.model.add_module('relu1_1',nn.ReLU())\n        self.model.add_module(\"maxpool1\",nn.MaxPool2d(2,2,0))\n        #32*63*63\n        self.model.add_module(\"batch1\",nn.BatchNorm2d(32))\n        \n        self.model.add_module('conv2_1',nn.Conv2d(32,64,3,1,0,bias=False))\n        #64*61*61\n        self.model.add_module('SiLU2_1',nn.SiLU())\n        self.model.add_module(\"maxpool2\",nn.MaxPool2d(2,2,0))\n        #64*30*30\n       \n        #全连接网络层,输入64*30*30,输出10个类别的概率\n        self.model.add_module('linear1',nn.Linear(64*30*30,256))\n        self.model.add_module('dropout1',nn.Dropout(0.5))\n        self.model.add_module('Leakyrelu3_1',nn.LeakyReLU())\n        self.model.add_module('linear2',nn.Linear(256,2))\n        self.model.add_module('Tanh3_2',nn.Tanh())\n\n    def forward(self,input): \n        output=input\n        for name,module in self.model.named_children():\n            if name==\"linear1\":\n                output=output.view(-1,64*30*30)\n            output=module(output)\n        return F.softmax(output,dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:24:12.628796Z","iopub.execute_input":"2023-04-15T07:24:12.629105Z","iopub.status.idle":"2023-04-15T07:24:12.640885Z","shell.execute_reply.started":"2023-04-15T07:24:12.629076Z","shell.execute_reply":"2023-04-15T07:24:12.639808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def weight_init(m):\n    #获取对象所属的类的名称\n    class_name=m.__class__.__name__\n    #当对象的name中出现\"conv\",也就是卷积操作\n    if class_name.find('conv')!=-1:\n        #对卷积核按照正态分布的均值和标准差随机初始化\n        m.weight.data.normal_(0,0.02)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:24:12.642403Z","iopub.execute_input":"2023-04-15T07:24:12.643041Z","iopub.status.idle":"2023-04-15T07:24:12.656655Z","shell.execute_reply.started":"2023-04-15T07:24:12.642995Z","shell.execute_reply":"2023-04-15T07:24:12.655790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#初始化神经网络\nnetC=CNN()\nnetC.apply(weight_init)\nprint(netC)\n#优化器\noptimizer=optim.Adam(netC.parameters(),lr=0.0002,betas=(0.5,0.999))\n#损失函数\ncriterion=nn.NLLLoss()#负对数似然损失函数，也是交叉熵损失函数的一种\n#训练周期为20次\nnum_epochs=20","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:24:12.657902Z","iopub.execute_input":"2023-04-15T07:24:12.658463Z","iopub.status.idle":"2023-04-15T07:24:12.794155Z","shell.execute_reply.started":"2023-04-15T07:24:12.658437Z","shell.execute_reply":"2023-04-15T07:24:12.793158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    num=[i for i in range(len(train_X))]\n    np.random.shuffle(num)\n    train_X=train_X[num]\n    train_y=train_y[num]\n    for i in range(16):\n        image=[]\n        label=[]\n        for j in range(500):\n            image.append(train_X[1000*i+j].reshape((3,128,128)))\n            label.append(train_y[1000*i+j])\n        #将数据转换成可以处理的张量格式\n        image=torch.Tensor(image)#.to(device)\n        label=torch.Tensor(label).long()#.to(device)\n        #训练\n        netC.train()\n        #将数据放进去训练\n        output=netC(image)#.to(device)\n        #计算每次的损失函数\n        error=criterion(output,label)#.to(device)\n        #反向传播\n        error.backward()\n        print(i)\n        #优化器进行优化(梯度下降,降低误差)\n        optimizer.step()\n        #将梯度清空\n        optimizer.zero_grad()\n        print(epoch,error)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:24:12.795333Z","iopub.execute_input":"2023-04-15T07:24:12.795750Z","iopub.status.idle":"2023-04-15T07:58:51.887723Z","shell.execute_reply.started":"2023-04-15T07:24:12.795723Z","shell.execute_reply":"2023-04-15T07:58:51.883109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y=[]\nfor i in range(len(train_X)):\n    pred=netC(torch.Tensor(train_X[i].reshape((1,3,128,128))))[0]#.to(device)\n    pred=np.argmax(pred.detach().cpu().numpy())\n    pred_y.append(pred)\n    if i%1000==0:\n        print(i)\nprint(\"训练集的准确率:\",np.sum(pred_y==train_y)/len(train_y),\"训练集总数为:\",len(train_y))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:58:51.896624Z","iopub.execute_input":"2023-04-15T07:58:51.897229Z","iopub.status.idle":"2023-04-15T08:01:08.084386Z","shell.execute_reply.started":"2023-04-15T07:58:51.897159Z","shell.execute_reply":"2023-04-15T08:01:08.082989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y=[]\nfor i in range(len(test_X)):\n    pred=netC(torch.Tensor(test_X[i].reshape((1,3,128,128))))[0]\n    pred=np.argmax(pred.detach().cpu().numpy())\n    pred_y.append(pred)\n    if i%1000==0:\n        print(i)\nprint(\"测试集的准确率:\",np.sum(pred_y==test_y)/len(test_y),\"测试集总数为:\",len(test_y))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:01:08.085917Z","iopub.execute_input":"2023-04-15T08:01:08.087207Z","iopub.status.idle":"2023-04-15T08:01:41.971877Z","shell.execute_reply.started":"2023-04-15T08:01:08.087175Z","shell.execute_reply":"2023-04-15T08:01:41.970324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(netC,\"catsdogs.pth\")","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:16:19.520003Z","iopub.execute_input":"2023-04-15T08:16:19.520413Z","iopub.status.idle":"2023-04-15T08:16:19.623974Z","shell.execute_reply.started":"2023-04-15T08:16:19.520379Z","shell.execute_reply":"2023-04-15T08:16:19.622443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_df[\"image_path\"][i],)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:01:42.053480Z","iopub.execute_input":"2023-04-15T08:01:42.053848Z","iopub.status.idle":"2023-04-15T08:01:42.066340Z","shell.execute_reply.started":"2023-04-15T08:01:42.053816Z","shell.execute_reply":"2023-04-15T08:01:42.064787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgcnt=12500\ndata=[]\nfor i in range(imgcnt):\n    img=Image.open(test_df[\"image_path\"][i])\n    new_img=transform(img)\n    data.append(new_img.detach().cpu().numpy())\n    if i%500==0:\n        print(i)\n\ndata=np.array(data)\npred_y=[0 for i in range(len(data))]\nfor i in range(len(data)):\n    pred=netC(torch.Tensor(data[i].reshape((1,3,128,128))))[0]#.to(device))[0]\n    pred=np.argmax(pred.detach().cpu().numpy())\n    t=test_df[\"image_path\"][i].split('/')\n    t=t[len(t)-1]\n    t=int(t[:len(t)-4])\n    pred_y[t-1]=pred\n    if i%1000==0:\n        print(i)\n\n#写入文件\nimport csv\nhead=[\"id\",\"label\"]\ndata=[]\nfor i in range(len(pred_y)):\n    data.append([(i+1),pred_y[i]])\nwith open(\"answer.csv\",'w',newline='') as f:\n    writer=csv.writer(f)\n    writer.writerow(head)\n    writer.writerows(data)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:01:42.068081Z","iopub.execute_input":"2023-04-15T08:01:42.068399Z","iopub.status.idle":"2023-04-15T08:04:08.173190Z","shell.execute_reply.started":"2023-04-15T08:01:42.068368Z","shell.execute_reply":"2023-04-15T08:04:08.172086Z"},"trusted":true},"execution_count":null,"outputs":[]}]}