{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"},{"sourceId":7757218,"sourceType":"datasetVersion","datasetId":4456930}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/d2l-module')","metadata":{"id":"pq2IXjyNydhJ","outputId":"360a5e72-42e9-4de8-f612-db2122b3c432","execution":{"iopub.status.busy":"2024-03-05T05:19:03.453698Z","iopub.execute_input":"2024-03-05T05:19:03.454347Z","iopub.status.idle":"2024-03-05T05:19:03.466942Z","shell.execute_reply.started":"2024-03-05T05:19:03.454314Z","shell.execute_reply":"2024-03-05T05:19:03.466251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport collections\nimport torch\nimport torchvision\nfrom torch import nn\nfrom datetime import datetime\nfrom d2l import torch as d2l\n\nsource_dir='/kaggle/input/dog-breed-identification'\ndata_dir='/kaggle/working/'","metadata":{"id":"1xyUiOwIyCYX","execution":{"iopub.status.busy":"2024-03-05T05:19:03.504157Z","iopub.execute_input":"2024-03-05T05:19:03.504405Z","iopub.status.idle":"2024-03-05T05:19:14.507482Z","shell.execute_reply.started":"2024-03-05T05:19:03.504384Z","shell.execute_reply":"2024-03-05T05:19:14.506529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_dog_data(source_dir,data_dir,valid_ratio):\n    \"\"\"分离训练集和测试集,并将验证集从原始的训练集中拆分出来\"\"\"\n    labels=d2l.read_csv_labels(os.path.join(source_dir,'labels.csv'))\n    n=collections.Counter(labels.values()).most_common()[-1][1] # 训练数据集中样本最少的类别中的样本数\n    n_valid_per_label=max(1,math.floor(n*valid_ratio))          # 验证集中每个类别的样本数\n    label_count={}\n    for train_file in os.listdir(os.path.join(source_dir,'train')):\n        label=labels[train_file.split('.')[0]]\n        fname=os.path.join(source_dir,'train',train_file)\n        d2l.copyfile(fname,os.path.join(data_dir,'train_valid_test','train_valid',label))\n        if label not in label_count or label_count[label]<n_valid_per_label:\n            d2l.copyfile(fname,os.path.join(data_dir,'train_valid_test','valid',label))\n            label_count[label]=label_count.get(label,0)+1\n        else:\n            d2l.copyfile(fname,os.path.join(data_dir,'train_valid_test','train',label))\n    for test_file in os.listdir(os.path.join(source_dir,'test')):\n        d2l.copyfile(os.path.join(source_dir,'test',test_file),os.path.join(data_dir,'train_valid_test','test','unknown'))\n        \nbatch_size=128\nreorg_dog_data(source_dir,data_dir,valid_ratio=0.1)","metadata":{"id":"wmbJsBD9yCYY","execution":{"iopub.status.busy":"2024-03-05T05:19:14.509163Z","iopub.execute_input":"2024-03-05T05:19:14.509589Z","iopub.status.idle":"2024-03-05T05:23:38.240585Z","shell.execute_reply.started":"2024-03-05T05:19:14.509564Z","shell.execute_reply":"2024-03-05T05:23:38.239517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train=torchvision.transforms.Compose([\n    torchvision.transforms.RandomResizedCrop(224,scale=(0.08,1.0),ratio=(3.0/4.0,4.0/3.0)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ColorJitter(brightness=0.4,contrast=0.4,saturation=0.4),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])])\n\ntransform_test=torchvision.transforms.Compose([\n    torchvision.transforms.Resize(256),\n    torchvision.transforms.CenterCrop(224),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])])\n\ntrain_ds,train_valid_ds=[torchvision.datasets.ImageFolder(os.path.join(data_dir,'train_valid_test',folder),transform=transform_train) for folder in ['train','train_valid']]\nvalid_ds,test_ds=[torchvision.datasets.ImageFolder(os.path.join(data_dir,'train_valid_test',folder),transform=transform_test) for folder in ['valid','test']]\n\ntrain_iter,train_valid_iter=[torch.utils.data.DataLoader(dataset,batch_size,shuffle=True,drop_last=True) for dataset in (train_ds,train_valid_ds)]\nvalid_iter=torch.utils.data.DataLoader(valid_ds,batch_size,shuffle=False,drop_last=True)\ntest_iter=torch.utils.data.DataLoader(test_ds,batch_size,shuffle=False,drop_last=False)","metadata":{"id":"_B5p2ImLyCYY","execution":{"iopub.status.busy":"2024-03-05T05:23:38.242029Z","iopub.execute_input":"2024-03-05T05:23:38.242507Z","iopub.status.idle":"2024-03-05T05:23:38.405386Z","shell.execute_reply.started":"2024-03-05T05:23:38.242474Z","shell.execute_reply":"2024-03-05T05:23:38.404583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_net(devices):\n    finetune_net=nn.Sequential()\n    finetune_net.features=torchvision.models.resnet34(pretrained=True)\n    # 定义一个新的输出网络,共有120个输出类别\n    finetune_net.output_new=nn.Sequential(nn.Linear(1000,256),nn.ReLU(),nn.Linear(256,120))\n    nn.init.kaiming_uniform_(finetune_net.output_new[0].weight)\n    nn.init.kaiming_uniform_(finetune_net.output_new[2].weight)\n    finetune_net=finetune_net.to(devices[0])\n    for params in finetune_net.features.parameters():\n        params.requires_grad=False  # 冻结参数\n    return finetune_net\n\nloss=nn.CrossEntropyLoss(reduction='none')\n\ndef train(net,train_iter,valid_iter,num_epochs,lr,wd,devices):\n    net=nn.DataParallel(net,device_ids=devices).to(devices[0])\n    trainer=torch.optim.SGD((param for param in net.parameters() if param.requires_grad),lr=lr,momentum=0.9,weight_decay=wd)\n    scheduler=torch.optim.lr_scheduler.MultiStepLR(trainer,[140,190],0.1)\n    timer,animator=d2l.Timer(),d2l.Animator(xlabel='epoch',xlim=[1,num_epochs],ylim=[0,1],legend=['train loss','train acc','valid acc'] if valid_iter is not None else ['train loss','train acc'])\n    for epoch in range(num_epochs):\n        net.train()\n        metric=d2l.Accumulator(3)\n        for i,(features,labels) in enumerate(train_iter):\n            timer.start()\n            l,acc=d2l.train_batch_ch13(net,features,labels,loss,trainer,devices)\n            metric.add(l,acc,labels.shape[0])\n            timer.stop()\n        animator.add(epoch+1,(metric[0]/metric[2],metric[1]/metric[2],None))\n        if valid_iter is not None:\n            valid_acc=d2l.evaluate_accuracy_gpu(net,valid_iter)\n            animator.add(epoch+1,(None,None,valid_acc))\n        scheduler.step()\n    print(f'train loss {metric[0]/metric[2]:.3f}, train acc {metric[1]/metric[2]:.3f}'+(f', valid acc {valid_acc:.3f}' if valid_iter is not None else '')+f'\\n{metric[2]*num_epochs/timer.sum():.1f} examples/sec on {str(devices)}')","metadata":{"id":"wth0tgg3yCYY","execution":{"iopub.status.busy":"2024-03-05T05:23:38.407123Z","iopub.execute_input":"2024-03-05T05:23:38.407402Z","iopub.status.idle":"2024-03-05T05:23:38.421236Z","shell.execute_reply.started":"2024-03-05T05:23:38.407378Z","shell.execute_reply":"2024-03-05T05:23:38.420265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"devices,num_epochs,lr,wd=d2l.try_all_gpus(),240,5e-4,5e-4\nnet=get_net(devices)\ntrain(net,train_iter,valid_iter,num_epochs,lr,wd,devices)\ntorch.save(net.state_dict(),os.path.join(data_dir,'resnet34-dog-breed-'+str(num_epochs)+'epochs-'+datetime.now().isoformat()+'.pth'))","metadata":{"id":"XXOIVOUXyCYZ","outputId":"7eb63796-aa26-4d66-9850-bcb3d78a1013","execution":{"iopub.status.busy":"2024-03-05T05:40:27.598230Z","iopub.execute_input":"2024-03-05T05:40:27.598601Z","iopub.status.idle":"2024-03-05T10:16:34.040137Z","shell.execute_reply.started":"2024-03-05T05:40:27.598574Z","shell.execute_reply":"2024-03-05T10:16:34.039048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#net=get_net(devices)\n#train(net,train_valid_iter,None,num_epochs,lr,wd,devices)\n#torch.save(net.state_dict(),os.path.join(data_dir,'resnet34-dog-breed-'+str(num_epochs)+'epochs-'+datetime.now().isoformat().+'pth'))\npreds=[]\nnet.eval()\nfor data,label in test_iter:\n    output=torch.nn.functional.softmax(net(data.to(devices[0])),dim=1)\n    preds.extend(output.cpu().detach().numpy())\nids=sorted(os.listdir(os.path.join(data_dir,'train_valid_test','test','unknown')))\nwith open(os.path.join(data_dir,'submission.csv'),'w') as f:\n    f.write('id,'+','.join(train_valid_ds.classes)+'\\n')\n    for i,output in zip(ids,preds):\n        f.write(i.split('.')[0]+','+','.join([str(num) for num in output])+'\\n')","metadata":{"id":"zG30NZ3vyCYa","outputId":"e08e243f-b09d-4890-9240-8e7deeb931ab","execution":{"iopub.status.busy":"2024-03-05T10:39:25.684072Z","iopub.execute_input":"2024-03-05T10:39:25.684974Z","iopub.status.idle":"2024-03-05T10:40:27.812838Z","shell.execute_reply.started":"2024-03-05T10:39:25.684938Z","shell.execute_reply":"2024-03-05T10:40:27.811705Z"},"trusted":true},"execution_count":null,"outputs":[]}]}