{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-04T06:16:39.900036Z","iopub.execute_input":"2023-04-04T06:16:39.900445Z","iopub.status.idle":"2023-04-04T06:16:39.936045Z","shell.execute_reply.started":"2023-04-04T06:16:39.900411Z","shell.execute_reply":"2023-04-04T06:16:39.935024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch\nimport torch.nn as nn\nimport cv2\nimport matplotlib.pyplot as plt\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom torchvision import transforms\nimport copy\nimport tqdm\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:27:37.921781Z","iopub.execute_input":"2023-04-04T08:27:37.922120Z","iopub.status.idle":"2023-04-04T08:27:41.294498Z","shell.execute_reply.started":"2023-04-04T08:27:37.922088Z","shell.execute_reply":"2023-04-04T08:27:41.292885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#生成文件夹\nimport shutil\nimport os\nfrom PIL import Image\nimport zipfile\ndef make_directory(dir_path):\n    if os.path.exists(dir_path):\n        shutil.rmtree(dir_path)\n    os.makedirs(dir_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:27:45.264875Z","iopub.execute_input":"2023-04-04T08:27:45.265679Z","iopub.status.idle":"2023-04-04T08:27:45.271751Z","shell.execute_reply.started":"2023-04-04T08:27:45.265639Z","shell.execute_reply":"2023-04-04T08:27:45.270297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Dataset_Path = '../Output/dogs-vs-cats'\nmake_directory(Dataset_Path)\n#抽取训练数据\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip','r') as zip_e:\n    zip_e.extractall(Dataset_Path)\n\n#抽取测试数据\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats/test1.zip','r')as zip_e2:\n    zip_e2.extractall(Dataset_Path)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:27:48.519761Z","iopub.execute_input":"2023-04-04T08:27:48.520653Z","iopub.status.idle":"2023-04-04T08:28:07.545466Z","shell.execute_reply.started":"2023-04-04T08:27:48.520614Z","shell.execute_reply":"2023-04-04T08:28:07.544336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_Data_Path = os.path.sep.join([Dataset_Path,'train']) #训练集文件夹路径\nTest_Data_Path  = os.path.sep.join([Dataset_Path,'test1']) #测试文件夹路径\ntrain_files = os.listdir(Train_Data_Path)   #训练文件夹下图像路径列表\ntest_files  = os.listdir(Test_Data_Path)    #测试文件夹下图像路径列表\nprint(len(train_files))\nprint(len(test_files))\n# print(train_files[0])\n# if 'dog' in train_files[0]:\n#     print(\"true\")","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:13.490875Z","iopub.execute_input":"2023-04-04T08:28:13.491244Z","iopub.status.idle":"2023-04-04T08:28:13.565301Z","shell.execute_reply.started":"2023-04-04T08:28:13.491213Z","shell.execute_reply":"2023-04-04T08:28:13.563738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.open(os.path.sep.join([Train_Data_Path,train_files[3]]))","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:20.119102Z","iopub.execute_input":"2023-04-04T08:28:20.119802Z","iopub.status.idle":"2023-04-04T08:28:20.195141Z","shell.execute_reply.started":"2023-04-04T08:28:20.119762Z","shell.execute_reply":"2023-04-04T08:28:20.194143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatVsDogsDataset(Dataset):\n    def __init__(self,file_list,dir,mode='train',transform = None):\n        self.file_list = file_list   #图像列表\n        self.dir = dir          #图像列表上一级路劲\n        self.mode = mode\n        self.transform = transform\n        if self.mode == 'train':\n            if 'dog' in self.file_list[0]:  #file_list?\n                self.label = 1\n            else:\n                self.label = 0\n        \n    def __len__(self):\n        return len(self.file_list)\n        \n        \n    def __getitem__(self,idx):\n        img = Image.open(os.path.sep.join([self.dir,self.file_list[idx]]))\n        if self.transform:\n            img = self.transform(img)\n        if self.mode == 'train':\n            img = img.numpy()\n            return img.astype('float32'),self.label\n        else:\n            img = img.numpy()\n            return img.astype('float32'),self.file_list[idx]  ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:23.557943Z","iopub.execute_input":"2023-04-04T08:28:23.558544Z","iopub.status.idle":"2023-04-04T08:28:23.567071Z","shell.execute_reply.started":"2023-04-04T08:28:23.558506Z","shell.execute_reply":"2023-04-04T08:28:23.565883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transform = transforms.Compose([\n    transforms.Resize(256),\n    transforms.ColorJitter(),\n    transforms.RandomCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.Resize(128),\n    transforms.ToTensor()\n])","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:27.760677Z","iopub.execute_input":"2023-04-04T08:28:27.761360Z","iopub.status.idle":"2023-04-04T08:28:27.766855Z","shell.execute_reply.started":"2023-04-04T08:28:27.761323Z","shell.execute_reply":"2023-04-04T08:28:27.765685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_files = []\nfor tf in train_files:\n    #print(\"cat\",tf)\n    if 'cat' in tf:\n        cat_files.append(tf)\ndog_files =[]\nfor tf in train_files:\n    if 'dog' in tf:\n        dog_files.append(tf)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:30.511513Z","iopub.execute_input":"2023-04-04T08:28:30.511902Z","iopub.status.idle":"2023-04-04T08:28:30.525719Z","shell.execute_reply.started":"2023-04-04T08:28:30.511870Z","shell.execute_reply":"2023-04-04T08:28:30.524625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(cat_files))\nprint(len(dog_files))","metadata":{"execution":{"iopub.status.busy":"2023-04-04T07:16:32.318955Z","iopub.execute_input":"2023-04-04T07:16:32.320905Z","iopub.status.idle":"2023-04-04T07:16:32.328408Z","shell.execute_reply.started":"2023-04-04T07:16:32.320850Z","shell.execute_reply":"2023-04-04T07:16:32.327189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def __init__(self,file_list,dir,mode='train',transform = None):\n#数据预处理\ncats = CatVsDogsDataset(cat_files,Train_Data_Path,transform = data_transform)\ndogs = CatVsDogsDataset(dog_files,Train_Data_Path,transform = data_transform)\ncatdogs = torch.utils.data.ConcatDataset([cats,dogs])","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:34.983402Z","iopub.execute_input":"2023-04-04T08:28:34.984136Z","iopub.status.idle":"2023-04-04T08:28:34.989766Z","shell.execute_reply.started":"2023-04-04T08:28:34.984097Z","shell.execute_reply":"2023-04-04T08:28:34.988732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#写dataloader\ndataloader = DataLoader(catdogs,batch_size = 32,shuffle=True,num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:40.153109Z","iopub.execute_input":"2023-04-04T08:28:40.154065Z","iopub.status.idle":"2023-04-04T08:28:40.163877Z","shell.execute_reply.started":"2023-04-04T08:28:40.154024Z","shell.execute_reply":"2023-04-04T08:28:40.161912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples,labels = iter(dataloader).__next__()\nprint(samples.shape) #[batchsize,channels,width,height]\ngrid_img = torchvision.utils.make_grid(samples[:16]) \n#做成一张照片torch.Size([3, 262, 1042])\n\nnp_grid_imgs = grid_img.numpy()#(3, 262, 1042)\n#np.transpose进行转换 tensor格式图片是[channels,width,height]\n#需要转换成[width,height,channels]\nplt.imshow(np.transpose(np_grid_imgs,(1,2,0)))","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:49.955323Z","iopub.execute_input":"2023-04-04T08:28:49.956051Z","iopub.status.idle":"2023-04-04T08:28:51.483436Z","shell.execute_reply.started":"2023-04-04T08:28:49.956007Z","shell.execute_reply":"2023-04-04T08:28:51.482397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#进行迁移学习\ndevice = 'cuda'\nmodel = torchvision.models.resnet18(pretrained=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:28:55.840247Z","iopub.execute_input":"2023-04-04T08:28:55.840762Z","iopub.status.idle":"2023-04-04T08:28:56.770088Z","shell.execute_reply.started":"2023-04-04T08:28:55.840724Z","shell.execute_reply":"2023-04-04T08:28:56.769076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:26:13.217588Z","iopub.status.idle":"2023-04-04T08:26:13.218274Z","shell.execute_reply.started":"2023-04-04T08:26:13.217997Z","shell.execute_reply":"2023-04-04T08:26:13.218031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fc = nn.Linear(512,2)\nmodel = model.to(device)\ncriterion = nn.CrossEntropyLoss()\noptim = torch.optim.Adam(model.parameters(),lr=0.002,amsgrad=True)\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optim, milestones=[500,1000,1500],gamma=0.5)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:29:02.185394Z","iopub.execute_input":"2023-04-04T08:29:02.186127Z","iopub.status.idle":"2023-04-04T08:29:05.469915Z","shell.execute_reply.started":"2023-04-04T08:29:02.186085Z","shell.execute_reply":"2023-04-04T08:29:05.468615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 3\nitr = 1\np_itr =200\nmodel.train()  #模型训练打开\ntotal_loss = 0\nloss_list =[]\nacc_list = []\nfor epoch in range(epochs):\n    for samples,labels in dataloader:\n        samples,labels = samples.to(device),labels.to(device)  #数据放到GPU上\n        optim.zero_grad()    #梯度置零\n        output = model(samples)#样本向前传输，得到预测结果\n        loss = criterion(output,labels) #预测结果和标签机损损失，本处交叉熵是CrossEntropyLoss\n        loss.backward()\n        optim.step()\n        total_loss += loss.item()\n        scheduler.step()\n        \n        if itr%p_itr == 0:\n            pred = torch.argmax(output, dim=1)\n            correct = pred.eq(labels)\n            acc = torch.mean(correct.float())\n            print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, Accuracy: {:.3f}'.format(epoch+1, epochs, itr, total_loss/p_itr, acc))\n            loss_list.append(total_loss/p_itr)\n            acc_list.append(acc.cpu().numpy())\n            total_loss = 0\n        itr +=1\nplt.plot(loss_list, label='loss')\nplt.plot(acc_list, label='accuracy')\nplt.legend()\nplt.title('training loss and accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:39:00.631663Z","iopub.execute_input":"2023-04-04T08:39:00.632955Z","iopub.status.idle":"2023-04-04T08:58:38.083486Z","shell.execute_reply.started":"2023-04-04T08:39:00.632906Z","shell.execute_reply":"2023-04-04T08:58:38.080694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename_path = 'ckpt_resnet18_catdog.pth'\ntorch.save(model.state_dict(),filename_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T08:59:42.767975Z","iopub.execute_input":"2023-04-04T08:59:42.769259Z","iopub.status.idle":"2023-04-04T08:59:42.888278Z","shell.execute_reply.started":"2023-04-04T08:59:42.769212Z","shell.execute_reply":"2023-04-04T08:59:42.887056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_transform = transforms.Compose([\n    transforms.Resize((128,128)),\n    transforms.ToTensor()\n])\ntestset = CatVsDogsDataset(test_files,Test_Data_Path,mode='test',transform = test_transform)\ntestloader = DataLoader(testset,batch_size=32,shuffle=False,num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T09:06:07.844502Z","iopub.execute_input":"2023-04-04T09:06:07.845582Z","iopub.status.idle":"2023-04-04T09:06:07.854579Z","shell.execute_reply.started":"2023-04-04T09:06:07.845516Z","shell.execute_reply":"2023-04-04T09:06:07.853167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nfn_list = []#?\npred_list = []\nfor x ,fn in testloader:\n    with torch.no_grad():\n        x = x.to(device)\n        output = model(x)\n        pred = torch.argmax(output,dim=1)\n        for n in fn:\n            fn_list.append(n[:-4])\n        for p in pred:\n            pred_list.append(p.item())\nsubmission = pd.DataFrame({'id':fn_list,'label':pred_list})\nsubmission.to_csv('pred_resnet18.csv',index = False)\nprint(\"test done\")","metadata":{"execution":{"iopub.status.busy":"2023-04-04T09:13:45.034010Z","iopub.execute_input":"2023-04-04T09:13:45.034390Z","iopub.status.idle":"2023-04-04T09:14:24.259062Z","shell.execute_reply.started":"2023-04-04T09:13:45.034355Z","shell.execute_reply":"2023-04-04T09:14:24.257764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples, _ = iter(testloader).__next__()\nprint(samples.shape)\nsamples = samples.to(device)\nfig = plt.figure(figsize=(24, 16))\nfig.tight_layout()\noutput = model(samples[:24])\npred = torch.argmax(output, dim=1)\npred = [p.item() for p in pred]\nad = {0:'cat', 1:'dog'}\nfor num, sample in enumerate(samples[:24]):\n    plt.subplot(4,6,num+1)\n    plt.title(ad[pred[num]])\n    plt.axis('off')\n    sample = sample.cpu().numpy()\n    plt.imshow(np.transpose(sample, (1,2,0)))","metadata":{"execution":{"iopub.status.busy":"2023-04-04T09:17:39.317305Z","iopub.execute_input":"2023-04-04T09:17:39.318047Z","iopub.status.idle":"2023-04-04T09:17:42.047848Z","shell.execute_reply.started":"2023-04-04T09:17:39.318008Z","shell.execute_reply":"2023-04-04T09:17:42.044579Z"},"trusted":true},"execution_count":null,"outputs":[]}]}