{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd#导入csv文件的库\nimport numpy as np#进行矩阵运算的库\nimport matplotlib.pyplot as plt#作图的库\nimport torch#一个深度学习的库Pytorch\nimport torch.nn as nn#neural network,神经网络\nfrom torch.autograd import Variable#从自动求导中引入变量\nimport torch.optim as optim#一个实现了各种优化算法的库\nimport torch.nn.functional as F#神经网络函数库\nimport os#与操作系统交互,处理文件和目录、管理进程、获取环境变量\nfrom PIL import Image, ImageOps, ImageFilter, ImageEnhance#PIL是图像处理库\nimport torchvision.transforms as transforms#图像、视频、文本增强和预处理的库\nimport zipfile#对ZIP格式的归档文件进行读取、写入和操作\nimport warnings#避免一些可以忽略的报错\nwarnings.filterwarnings('ignore')#filterwarnings()方法是用于设置警告过滤器的方法，它可以控制警告信息的输出方式和级别。","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-25T01:24:51.047688Z","iopub.execute_input":"2023-09-25T01:24:51.048162Z","iopub.status.idle":"2023-09-25T01:24:54.684940Z","shell.execute_reply.started":"2023-09-25T01:24:51.048121Z","shell.execute_reply":"2023-09-25T01:24:54.684000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir('../input/dogs-vs-cats-redux-kernels-edition'))#返回指定路径下的文件和目录名的列表\nzip_files = ['test', 'train']\n\nfor zip_file in zip_files:\n    with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}.zip\".format(zip_file),\"r\") as z:\n        z.extractall(\".\")#解压\n        print(\"{} unzipped\".format(zip_file))","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:24:54.687013Z","iopub.execute_input":"2023-09-25T01:24:54.687543Z","iopub.status.idle":"2023-09-25T01:25:10.133565Z","shell.execute_reply.started":"2023-09-25T01:24:54.687509Z","shell.execute_reply":"2023-09-25T01:25:10.132442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_Folder_Path = \"../working/train\"\nTest_Folder_Path = \"../working/test\"\ntrain_file_names = os.listdir(Train_Folder_Path)\ntest_file_names = os.listdir(Test_Folder_Path)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.135409Z","iopub.execute_input":"2023-09-25T01:25:10.136162Z","iopub.status.idle":"2023-09-25T01:25:10.165609Z","shell.execute_reply.started":"2023-09-25T01:25:10.136123Z","shell.execute_reply":"2023-09-25T01:25:10.164648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\nfull_paths = []\nfor file_name in train_file_names:\n    target = file_name.split(\".\")[0]\n    full_path = os.path.join(Train_Folder_Path, file_name)\n    full_paths.append(full_path)\n    targets.append(target)\ntrain_df = pd.DataFrame({\"image_path\":full_paths,\"target\":targets})\nprint(f\"len(train_df):{len(train_df)}\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.168615Z","iopub.execute_input":"2023-09-25T01:25:10.168989Z","iopub.status.idle":"2023-09-25T01:25:10.269927Z","shell.execute_reply.started":"2023-09-25T01:25:10.168959Z","shell.execute_reply":"2023-09-25T01:25:10.268831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_paths = []\nfor file_name in test_file_names:\n    full_path = os.path.join(Test_Folder_Path, file_name)\n    full_paths.append(full_path)\n\ntest_df = pd.DataFrame({\"image_path\":full_paths})\nprint(f\"len(test_df):{len(test_df)}\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.271701Z","iopub.execute_input":"2023-09-25T01:25:10.272449Z","iopub.status.idle":"2023-09-25T01:25:10.321422Z","shell.execute_reply.started":"2023-09-25T01:25:10.272407Z","shell.execute_reply":"2023-09-25T01:25:10.320411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#定义正则化的标准，3个颜色通道的均值和方差,这是从ImageNet数据集上得出的数值\nnormalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\ntransform = transforms.Compose([\n    transforms.RandomRotation(30), #在[-30,30]的范围内随机旋转一个角度\n    transforms.RandomResizedCrop(size=128, scale=(0.8, 1.2),interpolation=transforms.functional.InterpolationMode.BILINEAR),#先将图像随机进行缩放操作,然后再将图像变成128*128的图像\n    transforms.RandomHorizontalFlip(), # 随机水平翻转\n    transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5), # 随机颜色变换\n    transforms.ToTensor(),#转换成张量形式\n    normalize#标准化操作\n])","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.323060Z","iopub.execute_input":"2023-09-25T01:25:10.323890Z","iopub.status.idle":"2023-09-25T01:25:10.332614Z","shell.execute_reply.started":"2023-09-25T01:25:10.323854Z","shell.execute_reply":"2023-09-25T01:25:10.331592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgcnt=10000\ndataX=[]\ndatay=[]\nfor i in range(imgcnt):\n    img=Image.open(train_df[\"image_path\"][i])#读取图片\n    new_img=transform(img)\n    dataX.append(new_img.detach().cpu().numpy())\n    datay.append(train_df['target'][i]==\"dog\")#狗是1,猫是0\n    if i%1000==0:\n        print(i)\ndataX=np.array(dataX)\ndatay=np.array(datay)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.334111Z","iopub.execute_input":"2023-09-25T01:25:10.334601Z","iopub.status.idle":"2023-09-25T01:25:10.926487Z","shell.execute_reply.started":"2023-09-25T01:25:10.334565Z","shell.execute_reply":"2023-09-25T01:25:10.925445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #划分训练集和测试集的函数\ndef train_test_split(dataX,datay,shuffle=True,percentage=0.8):\n    \"\"\"\n    将训练数据X和标签y以numpy.array数组的形式传入\n    划分的比例定为训练集:测试集=8:2\n    \"\"\"\n    if shuffle:\n        random_num=[index for index in range(len(dataX))]\n        np.random.shuffle(random_num)\n        dataX=dataX[random_num]\n        datay=datay[random_num]\n    split_num=int(len(dataX)*percentage)\n    train_X=dataX[:split_num]\n    train_y=datay[:split_num]\n    test_X=dataX[split_num:]\n    test_y=datay[split_num:]\n    return train_X,train_y,test_X,test_y\ntrain_X,train_y,valid_X,valid_y=train_test_split(dataX,datay,percentage=0.9)\nprint(f\"train_X.shape:{train_X.shape},valid_X.shape:{valid_X.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.927903Z","iopub.execute_input":"2023-09-25T01:25:10.928528Z","iopub.status.idle":"2023-09-25T01:25:10.947500Z","shell.execute_reply.started":"2023-09-25T01:25:10.928491Z","shell.execute_reply":"2023-09-25T01:25:10.946279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset:\n    #储存数据集的信息\n    def __init__(self,raw_data):\n        self.raw_data=raw_data\n    #获取数据集的长度\n    def __len__(self):\n        return len(self.raw_data)\n    #获取数据集某特定条目的内容\n    def __getitem__(self,index):\n        \n        image,label=self.raw_data[index]\n        \n        return image,label\n\nclass DataLoader:\n    #从数据集中随机加载数据,并拼接成一个batch.\n    def __init__(self,dataset,batch_size=1,shuffle=False):\n        self.dataset=dataset\n        self.batch_size=batch_size\n        self.shuffle=False\n    #实现迭代器,使用时,迭代获取数据内容\n    def __iter__(self):\n        #每次准备迭代的时候,打乱数据,并清空指针\n        self.cursor=0\n        #正常序列\n        self.indexs=np.arange(len(self.dataset))\n\n        #打乱序列\n        if self.shuffle:\n            np.random.shuffle(self.indexs)\n            \n        return self\n\n    def __next__(self):\n        #预期在这里返回一个batch数据,batch是从dataset中随机抓取的\n        begin=self.cursor\n        end=self.cursor+self.batch_size\n\n        if end >len(self.dataset):\n            #如果end超出长度了,那迭代到底了\n            raise StopIteration()#抛出异常\n        #更新指针\n        self.cursor=end\n        #选择begin到end的部分\n        batched_data=[]\n        for index in self.indexs[begin:end]:\n            item=self.dataset[index]\n            batched_data.append(item)\n        return batched_data","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.949075Z","iopub.execute_input":"2023-09-25T01:25:10.950066Z","iopub.status.idle":"2023-09-25T01:25:10.960850Z","shell.execute_reply.started":"2023-09-25T01:25:10.950039Z","shell.execute_reply":"2023-09-25T01:25:10.959740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=20\n\ntrain_data=np.array([[train_X[index],train_y[index]] for index in range(len(train_X))])\nvalid_data=np.array([[valid_X[index],valid_y[index]] for index in range(len(valid_X))])\n\ntrain_dataset=Dataset(train_data)\nvalid_dataset=Dataset(valid_data)\n\ntrain_loader=DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\nvalid_loader=DataLoader(valid_dataset,batch_size=batch_size,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.964577Z","iopub.execute_input":"2023-09-25T01:25:10.964911Z","iopub.status.idle":"2023-09-25T01:25:10.974412Z","shell.execute_reply.started":"2023-09-25T01:25:10.964887Z","shell.execute_reply":"2023-09-25T01:25:10.973438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n\n    def __init__(self):\n        \n        super(CNN,self).__init__()\n       \n        self.conv=nn.Sequential(\n        #3*128*128\n        nn.Conv2d(3,32,3,1,0,bias=False),\n        nn.BatchNorm2d(32),\n        #32*126*126\n        nn.MaxPool2d(2,2,0),\n        #32*63*63\n        nn.GELU(),\n        #64*60*60\n        nn.Conv2d(32,64,4,1,0,bias=False),\n        nn.BatchNorm2d(64),\n        #64*30*30\n        nn.MaxPool2d(2,2,0),\n        nn.GELU(),\n        #128*28*28\n        nn.Conv2d(64,128,3,1,0,bias=False),\n        nn.BatchNorm2d(128),\n        #128*14*14\n        nn.MaxPool2d(2,2,0),\n        nn.GELU(),\n        )\n        \n        self.fc=nn.Sequential(\n        nn.Linear(128*14*14,128),\n        nn.BatchNorm1d(128),\n        nn.GELU(),\n        nn.Linear(128,2)\n        )\n\n    def forward(self,x): \n        x=self.conv(x)\n        x=x.reshape(-1,128*14*14)\n        x=self.fc(x)\n        return F.log_softmax(x,dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.975966Z","iopub.execute_input":"2023-09-25T01:25:10.976359Z","iopub.status.idle":"2023-09-25T01:25:10.988195Z","shell.execute_reply.started":"2023-09-25T01:25:10.976324Z","shell.execute_reply":"2023-09-25T01:25:10.987236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms#Pytorch下面的图像处理库,用于对图像进行数据增强\nfrom PIL import Image#图像处理库\n#定义数据增强\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),  # 随机水平翻转\n    #transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1),  # 随机调整图像亮度、对比度、饱和度和色调\n    #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),  # 对图像进行标准化操作\n    #transforms.ToTensor(),  # 转换为张量\n])\n\n\ndef get_batch_transform(batch_data,is_train=True):\n    \"\"\"\n    input:batch_data batch*(image,label)\n          numpy.array\n    output,tensor:X,y\n    \"\"\"\n    X=[]\n    y=[]\n    batch_size=len(batch_data)\n    for i in range(batch_size):\n        dataX,datay=batch_data[i]\n        image=torch.Tensor(dataX)\n        if is_train:\n            image=train_transform(image)\n        X.append(image.cpu().numpy())\n        y.append(datay)\n    X=torch.Tensor(X)\n    y=torch.Tensor(y)\n    return X,y#返回经过处理后的数据X和y","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:10.991216Z","iopub.execute_input":"2023-09-25T01:25:10.991554Z","iopub.status.idle":"2023-09-25T01:25:11.003209Z","shell.execute_reply.started":"2023-09-25T01:25:10.991528Z","shell.execute_reply":"2023-09-25T01:25:11.001968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndevice='cuda' if torch.cuda.is_available() else\"cpu\"\nprint(f\"device:{device}\")\n\nnetC=CNN()\n#训练次数\nnum_epochs=50\n#优化器\noptimizer=optim.Adam(netC.parameters(),lr=0.00025,betas=(0.5,0.999))\n#损失函数\ncriterion=nn.NLLLoss()\n\nnetC.to(device)\n\nfor epoch in range(num_epochs):\n    #模型的训练\n    netC.train()\n    for index,batched_data in enumerate(train_loader):\n        X,y=get_batch_transform(batched_data,is_train=True)\n        X,y=X.to(device),y.long().to(device)\n        #将梯度清空\n        optimizer.zero_grad()\n        #将数据放进去训练\n        output=netC(X).to(device)\n        #计算每次的损失函数\n        loss=criterion(output,y).to(device)\n        #反向传播\n        loss.backward()\n        #优化器进行优化(梯度下降,降低误差)\n        optimizer.step()\n    print(f\"epoch:{epoch}\")\n    netC.eval()\n    with torch.no_grad():\n        train_acc=0\n        for index,batched_data in enumerate(train_loader):\n            X,y=get_batch_transform(batched_data,is_train=False)\n            X,y=X.to(device),y.long().to(device)\n            #将数据放进去训练\n            output=netC(X).to(device)\n            output=torch.argmax(output,dim=1)\n            train_acc+=torch.sum(output==y)/len(train_X)\n        valid_acc=0\n        for index,batched_data in enumerate(valid_loader):\n            X,y=get_batch_transform(batched_data,is_train=False)\n            X,y=X.to(device),y.long().to(device)\n            #将数据放进去训练\n            output=netC(X).to(device)\n            output=torch.argmax(output,dim=1)\n            valid_acc+=torch.sum(output==y)/len(valid_X)\n        print(f\"train_acc:{train_acc},valid_acc:{valid_acc}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:25:11.004902Z","iopub.execute_input":"2023-09-25T01:25:11.005358Z","iopub.status.idle":"2023-09-25T01:25:21.276462Z","shell.execute_reply.started":"2023-09-25T01:25:11.005325Z","shell.execute_reply":"2023-09-25T01:25:21.275325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#定义正则化的标准，3个颜色通道的均值和方差,这是从ImageNet数据集上得出的数值\nnormalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\ntransform = transforms.Compose([\n    transforms.RandomResizedCrop(size=128, scale=(0.8, 1.2),interpolation=transforms.functional.InterpolationMode.BILINEAR),#先将图像随机进行缩放操作,然后再将图像变成128*128的图像\n    transforms.ToTensor(),#转换成张量形式\n    normalize#标准化操作\n])\n\ntest_pred=[i for i in range(len(test_df))]\ntest_path=test_df['image_path'].values\nnetC.eval()\nfor i in range(len(test_path)):\n    image=Image.open(test_path[i])\n    image=transform(image)\n    index=int(test_path[i][16:-4])\n    pred=netC(image.unsqueeze(0).to(device))\n    pred=torch.argmax(pred)\n    test_pred[index-1]=pred.detach().cpu().numpy()\nprint(f\"test_pred[:10]:{test_pred[:10]}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:31:25.684076Z","iopub.execute_input":"2023-09-25T01:31:25.684456Z","iopub.status.idle":"2023-09-25T01:32:16.114400Z","shell.execute_reply.started":"2023-09-25T01:31:25.684425Z","shell.execute_reply":"2023-09-25T01:32:16.113327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv\")\nsubmission['label']=test_pred\nsubmission.to_csv(\"submission.csv\",index=None)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:32:52.230817Z","iopub.execute_input":"2023-09-25T01:32:52.231214Z","iopub.status.idle":"2023-09-25T01:32:52.336022Z","shell.execute_reply.started":"2023-09-25T01:32:52.231182Z","shell.execute_reply":"2023-09-25T01:32:52.334910Z"},"trusted":true},"execution_count":null,"outputs":[]}]}