{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Homework 6 - Berkeley STAT 157\n\n**Your name: XX, SID YY, teammates A,B,C (Please add your name, SID and teammates to ease Ryan and Rachel to grade.)**\n\nHandout 3/5/2019, due 3/12/2019 by 4pm. Please submit through gradescope.\n\nIn this homework, we will train a CNN model on CIFAR-10 and submit the results into [Kaggle](https://www.kaggle.com/c/cifar-10). The rule is similar to homework 4: \n\n- work as a team\n- submit your results into Kaggle\n- take a screen shot of your best score and insert it below\n- the top 3 teams/individuals will be awarded with 500 dollar AWS credits\n\nThe rest of this notebook contains a baseline ResNet-15 model to train on CIFAR-10. Please use it as a starting point. The end of this notebooks has several hints to improve your results.\n\nFirst, import the packages or modules required for the competition.","metadata":{"id":"O3yOyxG_Ej7E"}},{"cell_type":"code","source":"import torch\nfrom torch import nn,optim,tensor\nfrom torch.utils.data import DataLoader\nfrom torchvision.utils import make_grid\nfrom torchvision import datasets,transforms\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport time\nimport pandas as pd","metadata":{"attributes":{"classes":[],"id":"","n":"1"},"id":"l3MyZ1DTEj7I","execution":{"iopub.status.busy":"2021-07-17T06:17:50.932472Z","iopub.execute_input":"2021-07-17T06:17:50.932870Z","iopub.status.idle":"2021-07-17T06:17:52.258740Z","shell.execute_reply.started":"2021-07-17T06:17:50.932768Z","shell.execute_reply":"2021-07-17T06:17:52.257927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:17:59.393518Z","iopub.execute_input":"2021-07-17T06:17:59.393865Z","iopub.status.idle":"2021-07-17T06:18:10.470887Z","shell.execute_reply.started":"2021-07-17T06:17:59.393829Z","shell.execute_reply":"2021-07-17T06:18:10.469776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr\n\narchive = py7zr.SevenZipFile('../input/cifar-10/test.7z', mode='r')\n# Must have a root folder wrapper on the test dataset folder.\n# Otherwise, the ImageFolder will complain.\narchive.extractall(path=\"./root_test\")\narchive.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:18:17.234785Z","iopub.execute_input":"2021-07-17T06:18:17.235167Z","iopub.status.idle":"2021-07-17T06:34:47.179696Z","shell.execute_reply.started":"2021-07-17T06:18:17.235126Z","shell.execute_reply":"2021-07-17T06:34:47.178773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32 \nnum_print = int(50000//batch_size//4)  \nepoch_num = 50  \nlr = 0.01        \nstep_size = 10  ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:36:28.339682Z","iopub.execute_input":"2021-07-17T06:36:28.340054Z","iopub.status.idle":"2021-07-17T06:36:28.344596Z","shell.execute_reply.started":"2021-07-17T06:36:28.340021Z","shell.execute_reply":"2021-07-17T06:36:28.343764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transforms_RandomHorizontalFlip():\n    transform_train = transforms.Compose([transforms.RandomHorizontalFlip(),\n                                          transforms.ToTensor(),transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n    transform = transforms.Compose([transforms.ToTensor(),\n                                    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n    train_dataset = datasets.CIFAR10(root='./data', \n                                     train=True, transform = transform_train,download=True)\n    test_dataset = datasets.CIFAR10(root='./data', \n                                    train=False, transform = transform,download=True)\n    return train_dataset,test_dataset","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:36:30.425506Z","iopub.execute_input":"2021-07-17T06:36:30.425854Z","iopub.status.idle":"2021-07-17T06:36:30.432170Z","shell.execute_reply.started":"2021-07-17T06:36:30.425804Z","shell.execute_reply":"2021-07-17T06:36:30.431230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset,test_dataset = transforms_RandomHorizontalFlip()\n\ntrain_loader = DataLoader(train_dataset, batch_size = batch_size,shuffle=True)\ntest_loader = DataLoader(test_dataset, batch_size = batch_size,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:36:32.816147Z","iopub.execute_input":"2021-07-17T06:36:32.816476Z","iopub.status.idle":"2021-07-17T06:36:40.215908Z","shell.execute_reply.started":"2021-07-17T06:36:32.816449Z","shell.execute_reply":"2021-07-17T06:36:40.215018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ('plane', 'car', 'bird', 'cat','deer', 'dog', 'frog', 'horse', 'ship', 'truck')\ndef image_show(img):\n    img = img / 2 + 0.5     \n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\ndef label_show(loader):  \n    global classes\n    dataiter = iter(loader)  \n    images, labels = dataiter.next()\n    image_show(make_grid(images))\n    print(' '.join('%5s' % classes[labels[j]] for j in range(batch_size)))\n    return images,labels\nlabel_show(train_loader)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:37:00.733946Z","iopub.execute_input":"2021-07-17T06:37:00.734313Z","iopub.status.idle":"2021-07-17T06:37:01.050160Z","shell.execute_reply.started":"2021-07-17T06:37:00.734282Z","shell.execute_reply":"2021-07-17T06:37:01.049198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define the Model\n","metadata":{"id":"iMLgrJEDEj7l"}},{"cell_type":"code","source":"from torch import nn\n\ndef vgg_block(num_convs,in_channels,out_channels):\n    blk = []\n    for i in range(num_convs):\n        if i == 0:\n            blk.append(nn.Conv2d(in_channels,out_channels,kernel_size=3,stride = 1,padding = 1 ))\n        else:\n            blk.append(nn.Conv2d(out_channels,out_channels,kernel_size=3,stride = 1,padding = 1 ))\n        blk.append(nn.BatchNorm2d(out_channels))\n        blk.append(nn.ReLU(inplace = True))\n    blk.append(nn.MaxPool2d(kernel_size=2,stride=2))\n    return nn.Sequential(*blk)\n\nclass Vgg16_Net(nn.Module):\n    def __init__(self,conv_arch,fc_features,fc_hidden_units):\n        super(Vgg16_Net, self).__init__()\n        self.conv_arch = conv_arch\n        self.fc_features = fc_features\n        self.fc_hidden_units = fc_hidden_units\n        self.conv_layer = nn.Sequential()\n        for i ,(num_convs,in_channels,out_channels) in enumerate(self.conv_arch):\n            self.conv_layer.add_module('vgg_block_'+str(i+1),vgg_block(num_convs,in_channels,out_channels))\n        self.fc_layer = nn.Sequential(   \n                nn.Linear(self.fc_features, self.fc_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                \n                nn.Linear(self.fc_features, self.fc_hidden_units),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n        \n                nn.Linear(self.fc_hidden_units, 10)\n                )\n        \n    def forward(self, x):\n        x = self.conv_layer(x)\n        x = x.view(-1, self.fc_features)\n        x = self.fc_layer(x)\n        return x\n","metadata":{"attributes":{"classes":[],"id":"","n":"11"},"id":"OrffARNjEj7m","execution":{"iopub.status.busy":"2021-07-17T06:37:05.701736Z","iopub.execute_input":"2021-07-17T06:37:05.702064Z","iopub.status.idle":"2021-07-17T06:37:05.712456Z","shell.execute_reply.started":"2021-07-17T06:37:05.702035Z","shell.execute_reply":"2021-07-17T06:37:05.711646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Same Model in Another Way","metadata":{}},{"cell_type":"code","source":"'''\nfrom torch import nn\n\nclass Vgg16_Net(nn.Module):\n    def __init__(self):\n        super(Vgg16_Net, self).__init__()\n        #2个卷积层和1个最大池化层\n        self.layer1 = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size = 3, stride=1, padding=1),             # (32-3+2)/1+1 = 32  32*32*64\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(64,64, kernel_size = 3, stride=1, padding=1),             # (32-3+2)/1+1 = 32  32*32*64\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            \n            nn.MaxPool2d(2, 2)                                                  # (32-2)/2+1 = 16    16*16*64\n            \n            )\n        #2个卷积层和1个最大池化层\n        self.layer2 = nn.Sequential(\n            nn.Conv2d(64, 128, kernel_size = 3, stride=1, padding=1),           # (16-3+2)/1+1 = 16  16*16*128\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(128, 128, kernel_size = 3, stride=1, padding=1),          # (16-3+2)/1+1 = 16  16*16*128\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            \n            nn.MaxPool2d(2, 2)                                                  # (16-2)/2+1 = 8    8*8*128\n            )\n        #3个卷积层和1个最大池化层\n        self.layer3 = nn.Sequential(\n            nn.Conv2d(128, 256, kernel_size = 3, stride=1, padding=1),          # (8-3+2)/1+1 = 8  8*8*256\n            nn.BatchNorm2d(256),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(256, 256, kernel_size = 3, stride=1, padding=1),          # (8-3+2)/1+1 = 8  8*8*256\n            nn.BatchNorm2d(256),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(256, 256, kernel_size = 3, stride=1, padding=1),          # (8-3+2)/1+1 = 8  8*8*256\n            nn.BatchNorm2d(256),\n            nn.ReLU(inplace=True),\n            \n            nn.MaxPool2d(2, 2),                                                 # (8-2)/2+1 = 4    4*4*256\n            )\n        #3个卷积层和1个最大池化层\n        self.layer4 = nn.Sequential(\n            nn.Conv2d(256, 512, kernel_size = 3, stride=1, padding=1),          # (4-3+2)/1+1 = 4  4*4*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(512, 512, kernel_size = 3, stride=1, padding=1),          # (4-3+2)/1+1 = 4  4*4*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(512, 512, kernel_size = 3, stride=1, padding=1),          # (4-3+2)/1+1 = 4  4*4*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n            \n            nn.MaxPool2d(2, 2)                                                  # (4-2)/2+1 = 2    2*2*512\n            )\n        #3个卷积层和1个最大池化层\n        self.layer5 = nn.Sequential(\n            nn.Conv2d(512, 512, kernel_size = 3, stride=1, padding=1),          # (2-3+2)/1+1 = 2  2*2*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(512, 512, kernel_size = 3, stride=1, padding=1),          # (2-3+2)/1+1 = 2  2*2*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(512, 512, kernel_size = 3, stride=1, padding=1),          # (2-3+2)/1+1 = 2  2*2*512\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n            \n            nn.MaxPool2d(2, 2)                                                  # (2-2)/2+1 = 1    1*1*512\n            )\n        self.conv = nn.Sequential(\n            self.layer1,\n            self.layer2,\n            self.layer3,\n            self.layer4,\n            self.layer5\n            )\n        self.fc = nn.Sequential(    \n            nn.Linear(512, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n            \n            nn.Linear(512, 256),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n    \n            nn.Linear(256, 10)\n            )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = x.view(-1, 512)\n        x = self.fc(x)\n        return x\n\n'''","metadata":{"execution":{"iopub.status.busy":"2021-07-16T11:46:40.489408Z","iopub.execute_input":"2021-07-16T11:46:40.490437Z","iopub.status.idle":"2021-07-16T11:46:40.505412Z","shell.execute_reply.started":"2021-07-16T11:46:40.490391Z","shell.execute_reply":"2021-07-16T11:46:40.503922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and Optimizer","metadata":{"id":"YqGPcjd5Ej7z"}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:37:10.352681Z","iopub.execute_input":"2021-07-17T06:37:10.353292Z","iopub.status.idle":"2021-07-17T06:37:10.408459Z","shell.execute_reply.started":"2021-07-17T06:37:10.353246Z","shell.execute_reply":"2021-07-17T06:37:10.406662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_arch = ((2,3,64),(2,64,128),(3,128,256),(3,256,512),(3,512,512))\nfc_features = 512\nfc_hidden_units = 256\n\nmodel = Vgg16_Net(conv_arch,fc_features,fc_hidden_units).to(device)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:37:12.373863Z","iopub.execute_input":"2021-07-17T06:37:12.374184Z","iopub.status.idle":"2021-07-17T06:37:16.612350Z","shell.execute_reply.started":"2021-07-17T06:37:12.374157Z","shell.execute_reply":"2021-07-17T06:37:16.611491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(),lr = lr,momentum = 0.8,weight_decay = 0.001 )\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.5, last_epoch=-1)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:37:30.180838Z","iopub.execute_input":"2021-07-17T06:37:30.181179Z","iopub.status.idle":"2021-07-17T06:37:30.187603Z","shell.execute_reply.started":"2021-07-17T06:37:30.181150Z","shell.execute_reply":"2021-07-17T06:37:30.186393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train and Validate the Model\n","metadata":{"id":"cj3patV6Ej70"}},{"cell_type":"code","source":"loss_list = []\nstart = time.time()\n\n# train\nfor epoch in range(epoch_num):  \n    running_loss = 0.0\n    for i, (inputs, labels) in enumerate(train_loader, 0):\n        inputs ,labels = inputs.to(device),labels.to(device)\n        \n        optimizer.zero_grad()   \n        outputs = model(inputs)\n        loss = criterion(outputs, labels).to(device)\n        \n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        loss_list.append(loss.item())\n        if i % num_print == num_print-1 :\n            print('[%d epoch, %d] loss: %.6f' %(epoch + 1, i + 1, running_loss / num_print))\n            running_loss = 0.0  \n    lr_1 = optimizer.param_groups[0]['lr']\n    print('learn_rate : %.15f'%lr_1)\n    scheduler.step()\n\nend = time.time()\nprint('time:{}'.format(end-start))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T06:37:35.057726Z","iopub.execute_input":"2021-07-17T06:37:35.058105Z","iopub.status.idle":"2021-07-17T07:20:23.116496Z","shell.execute_reply.started":"2021-07-17T06:37:35.058077Z","shell.execute_reply":"2021-07-17T07:20:23.115676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss images show\nplt.plot(loss_list, label='Minibatch cost')\nplt.plot(np.convolve(loss_list,np.ones(200,)/200, mode='valid'),label='Running average')\nplt.ylabel('Cross Entropy')\nplt.xlabel('Iteration')\nplt.legend()\nplt.show()","metadata":{"attributes":{"classes":[],"id":"","n":"13"},"id":"OsMj9NkoEj70","outputId":"0d0ff2d1-27ea-4c55-aa89-b859395e2e47","execution":{"iopub.status.busy":"2021-07-17T07:20:50.926838Z","iopub.execute_input":"2021-07-17T07:20:50.927193Z","iopub.status.idle":"2021-07-17T07:20:51.226165Z","shell.execute_reply.started":"2021-07-17T07:20:50.927149Z","shell.execute_reply":"2021-07-17T07:20:51.225194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prediction with images\nimages,labels = label_show(test_loader)\nimages, labels = images.to(device), labels.to(device)\noutputs = model(images)\npredicted = outputs.argmax(dim = 1)\nprint('Predicted: ', ' '.join('%5s' % classes[predicted[j]] for j in range(batch_size)))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T07:20:53.661490Z","iopub.execute_input":"2021-07-17T07:20:53.661859Z","iopub.status.idle":"2021-07-17T07:20:53.872024Z","shell.execute_reply.started":"2021-07-17T07:20:53.661805Z","shell.execute_reply":"2021-07-17T07:20:53.870914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test\nmodel.eval()\ncorrect = 0.0\ntotal = 0\nwith torch.no_grad():  # No need to back propogate\n    for inputs, labels in test_loader:\n        inputs, labels = inputs.to(device), labels.to(device) \n        outputs = model(inputs)\n        pred = outputs.argmax(dim = 1)  \n        total += inputs.size(0)\n        correct += torch.eq(pred,labels).sum().item()\nprint('Accuracy of the network on the 10000 test images: %.2f %%' % (100.0 * correct / total))\n\nclass_correct = list(0. for i in range(10))\nclass_total = list(0. for i in range(10))\nfor inputs, labels in test_loader:\n    inputs, labels = inputs.to(device), labels.to(device)\n    outputs = model(inputs)\n    pred = outputs.argmax(dim = 1)  \n    c = (pred == labels.to(device)).squeeze()\n    for i in range(4):\n        label = labels[i]\n        class_correct[label] += float(c[i])\n        class_total[label] += 1\n\nfor i in range(10):\n    print('Accuracy of %5s : %.2f %%' % (classes[i], 100 * class_correct[i] / class_total[i]))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T07:20:59.828100Z","iopub.execute_input":"2021-07-17T07:20:59.828480Z","iopub.status.idle":"2021-07-17T07:21:09.416334Z","shell.execute_reply.started":"2021-07-17T07:20:59.828447Z","shell.execute_reply":"2021-07-17T07:21:09.415531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Map","metadata":{}},{"cell_type":"code","source":"# show feature_map\na = 0\ndef viz(module, input):\n    global a\n    x = input[0][0].cpu()\n    # print(x.device)\n    min_num = min(4,x.size()[0])\n    for i in range(min_num):\n        plt.subplot(1, min_num, i+1)\n        plt.xticks([]) \n        plt.yticks([])  \n        plt.axis('off')\t\n        plt.rcParams['figure.figsize'] = (20, 20) \n        plt.rcParams['savefig.dpi'] = 480\n        plt.rcParams['figure.dpi'] = 480\n        plt.imshow(x[i])\n    plt.savefig('./'+str(a)+'.jpg')\n    a += 1\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T07:21:17.894273Z","iopub.execute_input":"2021-07-17T07:21:17.894585Z","iopub.status.idle":"2021-07-17T07:21:17.901582Z","shell.execute_reply.started":"2021-07-17T07:21:17.894556Z","shell.execute_reply":"2021-07-17T07:21:17.900591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataiter = iter(test_loader)  \nimages, labels = dataiter.next()\n\nfor name, m in model.named_modules():\n    if isinstance(m, torch.nn.Conv2d):\n        m.register_forward_pre_hook(viz)\n\nmodel.eval()\nwith torch.no_grad():\n    model(images[2].unsqueeze(0).to(device))","metadata":{"execution":{"iopub.status.busy":"2021-07-16T12:34:06.262855Z","iopub.status.idle":"2021-07-16T12:34:06.263696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classify the Testing Set and Submit Results on Kaggle\n\nAfter obtaining a satisfactory model design and hyper-parameters, we use all training data sets (including validation sets) to retrain the model and classify the testing set.","metadata":{"id":"HA0IWeB3Ej71"}},{"cell_type":"code","source":"transform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])])\n\ntest_ds = datasets.ImageFolder('root_test', transform=transform_test)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, 128, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T07:21:29.883382Z","iopub.execute_input":"2021-07-17T07:21:29.883748Z","iopub.status.idle":"2021-07-17T07:21:31.548479Z","shell.execute_reply.started":"2021-07-17T07:21:29.883715Z","shell.execute_reply":"2021-07-17T07:21:31.547627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nlabels = {0:'airplane', 1:'automobile', 2:'bird', 3:'cat', 4:'deer', 5:'dog', 6:'frog', 7:'horse', 8:'ship', 9:'truck'}\n\nwith torch.no_grad():\n    for X, _ in test_iter:\n        y_hat = model(X.cuda(0))\n        preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())\nsorted_ids = list(range(1, len(test_ds) + 1))\nsorted_ids.sort(key=lambda x: str(x))  # this version of test dataset sortted by this order\ndf = pd.DataFrame({'id': sorted_ids, 'label': preds})\ndf['label'] = df['label'].apply(lambda x: labels[x])\ndf.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T07:21:46.416550Z","iopub.execute_input":"2021-07-17T07:21:46.416902Z","iopub.status.idle":"2021-07-17T07:24:31.698400Z","shell.execute_reply.started":"2021-07-17T07:21:46.416868Z","shell.execute_reply":"2021-07-17T07:24:31.697493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After executing the above code, we will get a \"submission.csv\" file. The format of this file is consistent with the Kaggle competition requirements. \n\n## Hints to Improve Your Results\n\n* You should use the compete CIFAR-10 dataset to get meaningful results. \n* You'd better use a GPU machine to run it, otherwise it'll be quite slow. (Please DON'T FORGET to stop or terminate your instance if you are not using it, otherwise AWS will change you)\n* Change the `batch_size` and number of epochs `num_epochs` to 128 and 100, respectively. (It will take a while to run.)\n* Change to another network, such as ResNet-34 or Inception","metadata":{"id":"DgmqEDpGEj72"}}]}