{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7752315,"sourceType":"datasetVersion","datasetId":4532685},{"sourceId":7757218,"sourceType":"datasetVersion","datasetId":4456930}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/d2l-module')","metadata":{"execution":{"iopub.status.busy":"2024-03-04T17:56:24.368470Z","iopub.execute_input":"2024-03-04T17:56:24.368826Z","iopub.status.idle":"2024-03-04T17:56:24.373106Z","shell.execute_reply.started":"2024-03-04T17:56:24.368796Z","shell.execute_reply":"2024-03-04T17:56:24.372184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nimport math\nimport os\nimport shutil\nimport pandas as pd\nimport torch\nimport torchvision\nfrom torch import nn\nfrom datetime import datetime\nfrom d2l import torch as d2l\n\nd2l.DATA_HUB['cifar10_tiny']=(d2l.DATA_URL+'kaggle_cifar10_tiny.zip','2068874e4b9a9f0fb07ebe0ad2b29754449ccacd')\ndemo=False   # 如果使用完整的Kaggle竞赛的数据集,设置demo为False\nif demo:\n    data_dir=d2l.download_extract('cifar10_tiny')\nelse:\n#     data_dir='../data/cifar-10/'\n    data_dir='/kaggle/input/cifar-10-object-recognition-in-images-organized/'","metadata":{"execution":{"iopub.status.busy":"2024-03-05T02:20:20.250827Z","iopub.execute_input":"2024-03-05T02:20:20.251216Z","iopub.status.idle":"2024-03-05T02:20:20.257532Z","shell.execute_reply.started":"2024-03-05T02:20:20.251182Z","shell.execute_reply":"2024-03-05T02:20:20.256610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname): #@save\n    \"\"\"读取fname来给标签字典返回一个文件名\"\"\"\n    with open(fname,'r') as f:\n        # 跳过文件头行(列名)\n        lines=f.readlines()[1:]\n    tokens=[l.rstrip().split(',') for l in lines]\n    return dict(((name,label) for name,label in tokens))\n\nlabels=read_csv_labels(os.path.join(data_dir,'trainLabels.csv'))\nprint('# 训练样本:',len(labels))\nprint('# 类别:',len(set(labels.values())))","metadata":{"execution":{"iopub.status.busy":"2024-03-04T17:56:32.007738Z","iopub.execute_input":"2024-03-04T17:56:32.008145Z","iopub.status.idle":"2024-03-04T17:56:32.079983Z","shell.execute_reply.started":"2024-03-04T17:56:32.008113Z","shell.execute_reply":"2024-03-04T17:56:32.079107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename,target_dir):  #@save\n    \"\"\"将文件复制到目标目录\"\"\"\n    os.makedirs(target_dir,exist_ok=True)\n    shutil.copy(filename,target_dir)\n\ndef reorg_train_valid(data_dir,labels,valid_ratio): #@save\n    \"\"\"将验证集从原始的训练集中拆分出来\"\"\"\n    n=collections.Counter(labels.values()).most_common()[-1][1] # 训练数据集中样本最少的类别中的样本数\n    n_valid_per_label=max(1,math.floor(n*valid_ratio))  # 验证集中每个类别的样本数\n    label_count={}\n    for train_file in os.listdir(os.path.join(data_dir,'train')):\n        label=labels[train_file.split('.')[0]]\n        fname=os.path.join(data_dir,'train',train_file)\n        copyfile(fname,os.path.join(data_dir,'train_valid_test','train_valid',label))\n        if label not in label_count or label_count[label]<n_valid_per_label:\n            copyfile(fname,os.path.join(data_dir,'train_valid_test','valid',label))\n            label_count[label]=label_count.get(label,0)+1\n        else:\n            copyfile(fname,os.path.join(data_dir,'train_valid_test','train',label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir): #@save\n    \"\"\"在预测期间整理测试集,以方便读取\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir,'test')):\n        copyfile(os.path.join(data_dir,'test',test_file),os.path.join(data_dir,'train_valid_test','test','unknown'))\n\ndef reorg_cifar10_data(data_dir,valid_ratio):\n    labels=read_csv_labels(os.path.join(data_dir,'trainLabels.csv'))\n    reorg_train_valid(data_dir,labels,valid_ratio)\n    reorg_test(data_dir)\n\nbatch_size=32 if demo else 128\nvalid_ratio=0.1\n# reorg_cifar10_data(data_dir,valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-03-04T17:56:32.081182Z","iopub.execute_input":"2024-03-04T17:56:32.081535Z","iopub.status.idle":"2024-03-04T17:56:32.093494Z","shell.execute_reply.started":"2024-03-04T17:56:32.081499Z","shell.execute_reply":"2024-03-04T17:56:32.092512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train=torchvision.transforms.Compose([\n    torchvision.transforms.Resize(40),\n    torchvision.transforms.RandomResizedCrop(32,scale=(0.64,1.0),ratio=(1.0,1.0)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914,0.4822,0.4465],[0.2023,0.1994,0.2010])\n])\ntransform_test=torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914,0.4822,0.4465],[0.2023,0.1994,0.2010])\n])\n\ntrain_ds,train_valid_ds=[torchvision.datasets.ImageFolder(os.path.join(data_dir,'train_valid_test',folder),transform=transform_train) for folder in ['train','train_valid']]\nvalid_ds,test_ds=[torchvision.datasets.ImageFolder(os.path.join(data_dir,'train_valid_test',folder),transform=transform_test) for folder in ['valid','test']]\n\ntrain_iter,train_valid_iter=[torch.utils.data.DataLoader(dataset,batch_size,shuffle=True,drop_last=True) for dataset in (train_ds,train_valid_ds)]\nvalid_iter=torch.utils.data.DataLoader(valid_ds,batch_size,shuffle=False,drop_last=True)\ntest_iter=torch.utils.data.DataLoader(test_ds,batch_size,shuffle=False,drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-04T17:56:32.094651Z","iopub.execute_input":"2024-03-04T17:56:32.094967Z","iopub.status.idle":"2024-03-04T18:02:16.923020Z","shell.execute_reply.started":"2024-03-04T17:56:32.094940Z","shell.execute_reply":"2024-03-04T18:02:16.922029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_net():\n    num_classes=10\n    net=d2l.resnet18(num_classes,3)\n    return net\n\nloss=nn.CrossEntropyLoss(reduction='none')\n\ndef train(net,train_iter,valid_iter,num_epochs,lr,wd,devices):\n    trainer=torch.optim.SGD(net.parameters(),lr=lr,momentum=0.9,weight_decay=wd)\n    scheduler=torch.optim.lr_scheduler.MultiStepLR(trainer,[140,190],0.1)\n    num_batches,timer=len(train_iter),d2l.Timer()\n    legend=['train loss','train acc']\n    if valid_iter is not None:\n        legend.append('valid acc')\n    animator=d2l.Animator(xlabel='epoch',xlim=[1,num_epochs],ylim=[0,1],legend=legend)\n    net=nn.DataParallel(net,device_ids=devices).to(devices[0])\n    for epoch in range(num_epochs):\n        net.train()\n        metric=d2l.Accumulator(3)\n        for i,(features,labels) in enumerate(train_iter):\n            timer.start()\n            l,acc=d2l.train_batch_ch13(net,features,labels,loss,trainer,devices)\n            metric.add(l,acc,labels.shape[0])\n            timer.stop()\n        animator.add(epoch+1,(metric[0]/metric[2],metric[1]/metric[2],None))\n        if valid_iter is not None:\n            valid_acc=d2l.evaluate_accuracy_gpu(net,valid_iter)\n            animator.add(epoch+1,(None,None,valid_acc))\n        scheduler.step()\n    measures=(f'train loss {metric[0]/metric[2]:.3f}, train acc {metric[1]/metric[2]:.3f}')\n    if valid_iter is not None:\n        measures+=f', valid acc {valid_acc:.3f}'\n    print(measures+f'\\n{metric[2]*num_epochs/timer.sum():.1f} examples/sec on {str(devices)}')","metadata":{"execution":{"iopub.status.busy":"2024-03-04T18:02:16.925624Z","iopub.execute_input":"2024-03-04T18:02:16.926055Z","iopub.status.idle":"2024-03-04T18:02:16.939945Z","shell.execute_reply.started":"2024-03-04T18:02:16.926020Z","shell.execute_reply":"2024-03-04T18:02:16.938940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"devices,num_epochs,lr,wd=d2l.try_all_gpus(),240,5e-4,5e-4\nnet=get_net()\ntrain(net,train_iter,valid_iter,num_epochs,lr,wd,devices)\ntorch.save(net.state_dict(),os.path.join(data_dir,'resnet18-cifar10-'+str(num_epochs)+'epochs-'+datetime.now().strftime(\"%Y%m%d%H%M%S\")+'.pth'))","metadata":{"execution":{"iopub.status.busy":"2024-03-04T18:12:11.033700Z","iopub.execute_input":"2024-03-04T18:12:11.034348Z","iopub.status.idle":"2024-03-05T01:19:43.495682Z","shell.execute_reply.started":"2024-03-04T18:12:11.034311Z","shell.execute_reply":"2024-03-05T01:19:43.492704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(net.state_dict(),'resnet18-cifar10-'+str(num_epochs)+'epochs-'+datetime.now().isoformat()+'.pth')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T02:24:58.546537Z","iopub.execute_input":"2024-03-05T02:24:58.547212Z","iopub.status.idle":"2024-03-05T02:24:58.619222Z","shell.execute_reply.started":"2024-03-05T02:24:58.547169Z","shell.execute_reply":"2024-03-05T02:24:58.618413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 太贵了，不在整个训练集上训练了\n#net=get_net()\n#train(net,train_valid_iter,None,num_epochs,lr,wd,devices,lr_period,lr_decay)\n#torch.save(net.state_dict(),os.path.join(data_dir,'resnet18-cifar10-'+str(num_epochs)+'epochs-'+datetime.now().strftime(\"%Y%m%d%H%M%S\")+'.pth'))\nnet.eval()\npreds=[]\nfor X,_ in test_iter:\n    y_hat=net(X.to(devices[0]))\n    preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())\nsorted_ids=list(range(1,len(test_ds)+1))\nsorted_ids.sort(key=lambda x:str(x))\ndf=pd.DataFrame({'id':sorted_ids,'label':preds})\ndf['label']=df['label'].apply(lambda x:train_valid_ds.classes[x])\n# df.to_csv(os.path.join(data_dir,'submission.csv'),index=False)\ndf.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T01:38:46.497336Z","iopub.execute_input":"2024-03-05T01:38:46.497676Z","iopub.status.idle":"2024-03-05T02:15:50.970353Z","shell.execute_reply.started":"2024-03-05T01:38:46.497652Z","shell.execute_reply":"2024-03-05T02:15:50.969530Z"},"trusted":true},"execution_count":null,"outputs":[]}]}