{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **树叶分类课程竞赛**\n- 首先要多谢Neko Kiku提供的baseline代码，思路非常清晰；\n- 本代码思想很简单，三个臭皮匠赛过诸葛亮，总共训练了3个优秀的模型（ResNeSt+ResNeXt+DenseNet）,最后进行集成，结果会更加鲁棒（公榜第12升到私榜第7也侧面反映了其鲁棒性）；\n- 代码是在本地计算机上跑的，由于Kaggle的运行时间有限制，无法分享运行完所有模型的结果，在这里我将我本地各个模型运行的结果附在了input文件夹里提供结果参考，以及方便走完代码整个流程；\n- 总结了图像分类任务的几个小技巧：\n1. 数据增强：特别是CutMix和预测时候对test样本进行TTA(Test Time Augmentation);\n2. 模型：可使用表现较好的预训练过的模型；\n3. 优化器：使用AdamW（对于含有L2正则项的优化，如weight decay），学习率采用cosine学习率CosineAnnealingLR;\n4. 交叉验证：使用K折交叉验证；","metadata":{"id":"ijYSEHmOf-62"}},{"cell_type":"code","source":"!pip install ttach\n# 安装TTA包","metadata":{"id":"BKct5uTcgFhY","outputId":"8837faea-48bd-4796-bc1b-a76bcd2e9ddd","execution":{"iopub.status.busy":"2021-06-28T04:06:50.384037Z","iopub.execute_input":"2021-06-28T04:06:50.384457Z","iopub.status.idle":"2021-06-28T04:06:58.736819Z","shell.execute_reply.started":"2021-06-28T04:06:50.384371Z","shell.execute_reply":"2021-06-28T04:06:58.735628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/ildoonet/cutmix \n# 安装CutMix","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:07:12.669137Z","iopub.execute_input":"2021-06-28T04:07:12.669527Z","iopub.status.idle":"2021-06-28T04:07:21.625824Z","shell.execute_reply.started":"2021-06-28T04:07:12.669469Z","shell.execute_reply":"2021-06-28T04:07:21.624993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 安装ResNeSt模型包\n!pip install resnest --pre","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:07:32.766716Z","iopub.execute_input":"2021-06-28T04:07:32.767081Z","iopub.status.idle":"2021-06-28T04:07:42.356778Z","shell.execute_reply.started":"2021-06-28T04:07:32.767043Z","shell.execute_reply":"2021-06-28T04:07:42.355538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 导入各种包\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nimport ttach as tta\nfrom resnest.torch import resnest50\n\nfrom cutmix.cutmix import CutMix\nfrom cutmix.utils import CutMixCrossEntropyLoss\n\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.model_selection import KFold\nfrom PIL import Image\nimport os\nimport matplotlib.pyplot as plt\nimport torchvision.models as models\n# This is for the progress bar.\nfrom tqdm import tqdm\n","metadata":{"id":"5GrL5NlBgoal","execution":{"iopub.status.busy":"2021-06-28T04:07:53.207773Z","iopub.execute_input":"2021-06-28T04:07:53.208153Z","iopub.status.idle":"2021-06-28T04:07:55.565071Z","shell.execute_reply.started":"2021-06-28T04:07:53.208110Z","shell.execute_reply":"2021-06-28T04:07:55.564094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **整理数据集**","metadata":{"id":"vLBrZv8rh6lE"}},{"cell_type":"markdown","source":"### **数据读取与预处理**","metadata":{"id":"zuLp3H0lCpHr"}},{"cell_type":"code","source":"# 看看label文件长啥样\nlabels_dataframe = pd.read_csv('../input/classify-leaves/train.csv')\nlabels_dataframe.head(5)","metadata":{"id":"yIRBT3qRClaZ","outputId":"cca948ce-4506-4b6b-e870-fc81b400924f","execution":{"iopub.status.busy":"2021-06-28T04:08:19.681005Z","iopub.execute_input":"2021-06-28T04:08:19.681387Z","iopub.status.idle":"2021-06-28T04:08:19.741419Z","shell.execute_reply.started":"2021-06-28T04:08:19.681353Z","shell.execute_reply":"2021-06-28T04:08:19.740428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 把label文件排个序\nleaves_labels = sorted(list(set(labels_dataframe['label'])))\nn_classes = len(leaves_labels)\nprint(n_classes)\nleaves_labels[:10]","metadata":{"id":"cVaLR5JiC_-o","outputId":"0a69cf0d-6467-428c-d819-d3c777bdb18c","execution":{"iopub.status.busy":"2021-06-28T04:08:21.869472Z","iopub.execute_input":"2021-06-28T04:08:21.869928Z","iopub.status.idle":"2021-06-28T04:08:21.882788Z","shell.execute_reply.started":"2021-06-28T04:08:21.869896Z","shell.execute_reply":"2021-06-28T04:08:21.881448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 把label转成对应的数字\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nclass_to_num","metadata":{"id":"tI2nHDzaDHfM","outputId":"a8801bfc-1d59-4890-9b74-f24ddbc90eaa","execution":{"iopub.status.busy":"2021-06-28T04:08:24.053191Z","iopub.execute_input":"2021-06-28T04:08:24.053604Z","iopub.status.idle":"2021-06-28T04:08:24.066931Z","shell.execute_reply.started":"2021-06-28T04:08:24.053554Z","shell.execute_reply":"2021-06-28T04:08:24.065858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 再转换回来，方便最后预测的时候使用\nnum_to_class = {v : k for k, v in class_to_num.items()}\nnum_to_class","metadata":{"id":"pL-bKsgqDKX4","outputId":"96ff1167-3b0c-4c49-a510-60f40330497d","execution":{"iopub.status.busy":"2021-06-28T04:08:29.726218Z","iopub.execute_input":"2021-06-28T04:08:29.726580Z","iopub.status.idle":"2021-06-28T04:08:29.738603Z","shell.execute_reply.started":"2021-06-28T04:08:29.726549Z","shell.execute_reply":"2021-06-28T04:08:29.737356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 继承pytorch的dataset，创建自己的\nclass TrainValidData(Dataset):\n    def __init__(self, csv_path, file_path, resize_height=224, resize_width=224, transform=None):\n        \"\"\"\n        Args:\n            csv_path (string): csv 文件路径\n            img_path (string): 图像文件所在路径\n\n        \"\"\"\n        \n        # 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#\n        self.resize_height = resize_height\n        self.resize_width = resize_width\n\n        self.file_path = file_path\n        self.to_tensor = transforms.ToTensor() #将数据转换成tensor形式\n        self.transform = transform\n\n        # 读取 csv 文件\n        # 利用pandas读取csv文件\n        self.data_info = pd.read_csv(csv_path, header=None)  #header=None是去掉表头部分\n        # 文件第一列包含图像文件名称\n        self.image_arr = np.asarray(self.data_info.iloc[1:,0]) #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始一直读取到最后一行\n        # 第二列是图像的 label\n        self.label_arr = np.asarray(self.data_info.iloc[1:,1])\n        # 计算 length\n        self.data_len = len(self.data_info.index) - 1\n\n    def __getitem__(self, index):\n        # 从 image_arr中得到索引对应的文件名\n        single_image_name = self.image_arr[index]\n\n        # 读取图像文件\n        img_as_img = Image.open(self.file_path + single_image_name)\n        \n        #如果需要将RGB三通道的图片转换成灰度图片可参考下面两行\n        # if img_as_img.mode != 'L':\n        #     img_as_img = img_as_img.convert('L')\n        \n        #设置好需要转换的变量，还可以包括一系列的nomarlize等等操作\n        transform = transforms.Compose([\n            transforms.Resize((224, 224)),\n            transforms.ToTensor()\n        ])\n        img_as_img = transform(img_as_img)\n\n        # 得到图像的 label\n        label = self.label_arr[index]\n        number_label = class_to_num[label]\n\n        return (img_as_img, number_label)  #返回每一个index对应的图片数据和对应的label\n\n    def __len__(self):\n        return self.data_len","metadata":{"id":"dyYerVWThdSX","execution":{"iopub.status.busy":"2021-06-28T04:08:34.030255Z","iopub.execute_input":"2021-06-28T04:08:34.030629Z","iopub.status.idle":"2021-06-28T04:08:34.042265Z","shell.execute_reply.started":"2021-06-28T04:08:34.030598Z","shell.execute_reply":"2021-06-28T04:08:34.041011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 继承pytorch的dataset，创建自己的\nclass TestData(Dataset):\n    def __init__(self, csv_path, file_path, resize_height=224, resize_width=224, transform = None):\n        \"\"\"\n        Args:\n            csv_path (string): csv 文件路径\n            img_path (string): 图像文件所在路径\n\n        \"\"\"\n        \n        # 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#\n        self.resize_height = resize_height\n        self.resize_width = resize_width\n\n        self.file_path = file_path\n        self.transform = transform\n        self.to_tensor = transforms.ToTensor() #将数据转换成tensor形式\n\n        # 读取 csv 文件\n        # 利用pandas读取csv文件\n        self.data_info = pd.read_csv(csv_path, header=None)  #header=None是去掉表头部分\n        # 文件第一列包含图像文件名称\n        self.image_arr = np.asarray(self.data_info.iloc[1:,0]) #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始一直读取到最后一行\n        # 计算 length\n        self.data_len = len(self.data_info.index) - 1\n        \n    def __getitem__(self, index):\n        # 从 image_arr中得到索引对应的文件名\n        single_image_name = self.image_arr[index]\n\n        # 读取图像文件\n        img_as_img = Image.open(self.file_path + single_image_name)\n        \n        #如果需要将RGB三通道的图片转换成灰度图片可参考下面两行\n        # if img_as_img.mode != 'L':\n        #     img_as_img = img_as_img.convert('L')\n        \n        #设置好需要转换的变量，还可以包括一系列的nomarlize等等操作\n        transform = transforms.Compose([\n            transforms.Resize((224, 224)),\n            transforms.ToTensor()\n        ])\n        img_as_img = transform(img_as_img)\n\n\n        return img_as_img\n\n    def __len__(self):\n        return self.data_len","metadata":{"id":"ckc_0XmovBHT","execution":{"iopub.status.busy":"2021-06-28T04:08:59.983643Z","iopub.execute_input":"2021-06-28T04:08:59.984047Z","iopub.status.idle":"2021-06-28T04:08:59.994125Z","shell.execute_reply.started":"2021-06-28T04:08:59.984010Z","shell.execute_reply":"2021-06-28T04:08:59.992907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    # 随机裁剪图像，所得图像为原始面积的0.08到1之间，高宽比在3/4和4/3之间。\n    # 然后，缩放图像以创建224 x 224的新图像\n    transforms.RandomResizedCrop(224, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)),\n    transforms.RandomHorizontalFlip(),\n    # 随机更改亮度，对比度和饱和度\n    transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),\n    # 添加随机噪声\n    transforms.ToTensor(),\n    # 标准化图像的每个通道\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])\nval_test_transform = transforms.Compose([\n    transforms.Resize(256),\n    # 从图像中心裁切224x224大小的图片\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])","metadata":{"id":"8W9yJbmsVuN_","execution":{"iopub.status.busy":"2021-06-28T04:09:03.943839Z","iopub.execute_input":"2021-06-28T04:09:03.944223Z","iopub.status.idle":"2021-06-28T04:09:03.951579Z","shell.execute_reply.started":"2021-06-28T04:09:03.944192Z","shell.execute_reply":"2021-06-28T04:09:03.950354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_val_path = '../input/classify-leaves/train.csv'\ntest_path = '../input/classify-leaves/test.csv'\n# csv文件中已经images的路径了，因此这里只到上一级目录\nimg_path = '../input/classify-leaves/'\n\ntrain_val_dataset = TrainValidData(train_val_path, img_path)\ntest_dataset = TestData(test_path, img_path, transform = val_test_transform)\nprint(train_val_dataset.data_info)\nprint(test_dataset.data_info)","metadata":{"id":"-y969Ozht3gI","outputId":"36224e70-3481-484f-f909-73d1c72bbca4","execution":{"iopub.status.busy":"2021-06-28T04:09:11.630227Z","iopub.execute_input":"2021-06-28T04:09:11.630611Z","iopub.status.idle":"2021-06-28T04:09:11.682046Z","shell.execute_reply.started":"2021-06-28T04:09:11.630579Z","shell.execute_reply":"2021-06-28T04:09:11.681184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **基于ResNeSt模型部分**","metadata":{}},{"cell_type":"markdown","source":"### **ResNeSt模型**","metadata":{"id":"nk1vuWbh-7Ry"}},{"cell_type":"code","source":"# 是否要冻住模型的前面一些层\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        model = model\n        for param in model.parameters():\n            param.requires_grad = False\n\n# ResNeSt模型\ndef resnest_model(num_classes, feature_extract = False):\n    model_ft = resnest50(pretrained=True)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))\n\n    return model_ft","metadata":{"id":"QflrcSQF-ouz","execution":{"iopub.status.busy":"2021-06-28T04:09:15.902682Z","iopub.execute_input":"2021-06-28T04:09:15.903169Z","iopub.status.idle":"2021-06-28T04:09:15.909184Z","shell.execute_reply.started":"2021-06-28T04:09:15.903138Z","shell.execute_reply":"2021-06-28T04:09:15.907947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 看一下是在cpu还是GPU上\ndef get_device():\n    return 'cuda' if torch.cuda.is_available() else 'cpu'\n\ndevice = get_device()\nprint(device)","metadata":{"id":"AoIkdlqo_ZKD","outputId":"123aeca7-95b4-4386-88c6-76d8df096d59","execution":{"iopub.status.busy":"2021-06-28T04:03:23.674675Z","iopub.status.idle":"2021-06-28T04:03:23.675083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"id":"jb4HXVVBmzEe","outputId":"f9e81580-f790-4607-cb30-3b05f4be84c6","execution":{"iopub.status.busy":"2021-06-28T04:03:23.675869Z","iopub.status.idle":"2021-06-28T04:03:23.676312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration options\nk_folds = 5\nnum_epochs = 30\nlearning_rate = 1e-4\nweight_decay = 1e-3\ntrain_loss_function = CutMixCrossEntropyLoss(True)\nvalid_loss_function = nn.CrossEntropyLoss()\n# For fold results\nresults = {}\n\n# Set fixed random number seed\ntorch.manual_seed(42)\n\n# Define the K-fold Cross Validator\nkfold = KFold(n_splits=k_folds, shuffle=True)","metadata":{"id":"hkkwMZyKuuKd","execution":{"iopub.status.busy":"2021-06-28T04:12:31.358819Z","iopub.execute_input":"2021-06-28T04:12:31.359401Z","iopub.status.idle":"2021-06-28T04:12:31.369570Z","shell.execute_reply.started":"2021-06-28T04:12:31.359363Z","shell.execute_reply":"2021-06-28T04:12:31.368525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **训练**","metadata":{"id":"TobMADIKCilB"}},{"cell_type":"code","source":"# Start print\nprint('--------------------------------------')\n\n# K-fold Cross Validation model evaluation\nfor fold, (train_ids,valid_ids) in enumerate(kfold.split(train_val_dataset)):\n\n  # Print\n  print(f'FOLD {fold}')\n  print('--------------------------------------')\n\n  # Sample elements randomly from a given list of ids, no replacement.\n  train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n  valid_subsampler = torch.utils.data.SubsetRandomSampler(valid_ids)\n\n  # Define data loaders for training and testing data in this fold\n  trainloader = torch.utils.data.DataLoader(\n                      CutMix(TrainValidData(train_val_path, img_path, transform = train_transform), num_class=176, beta=1.0, prob=0.5, num_mix=2), \n                      batch_size=32, sampler=train_subsampler, num_workers=0)\n  validloader = torch.utils.data.DataLoader(\n                      TrainValidData(train_val_path, img_path, transform = val_test_transform),\n                      batch_size=32, sampler=valid_subsampler, num_workers=0)\n  \n  # Initialize a model and put it on the device specified.\n  model = resnest_model(176)\n  model = model.to(device)\n  model.device = device\n  \n  # Initialize optimizer\n  optimizer = torch.optim.AdamW(model.parameters(),lr=learning_rate,weight_decay= weight_decay)\n  scheduler = CosineAnnealingLR(optimizer,T_max=10)\n\n  # Run the training loop for defined number of epochs\n  for epoch in range(0,num_epochs):\n    model.train()\n    # Print epoch\n    print(f'Starting epoch {epoch+1}')\n    # These are used to record information in training\n    train_losses = []\n    train_accs = []\n    # Iterate the training set by batches\n    for batch in tqdm(trainloader):\n      # Move images and labels to GPU\n      imgs, labels = batch\n      imgs = imgs.to(device)\n      labels = labels.to(device)\n      # Forward the data\n      logits = model(imgs)\n      # Calculate loss\n      loss = train_loss_function(logits,labels)\n      # Clear gradients in previous step\n      optimizer.zero_grad()\n      # Compute gradients for parameters\n      loss.backward()\n      # Update the parameters with computed gradients\n      optimizer.step()\n      # Compute the accuracy for current batch.\n      # acc = (logits.argmax(dim=-1) == labels).float().mean()\n      # Record the loss and accuracy.\n      train_losses.append(loss.item())\n      # train_accs.append(acc)\n    print(\"第%d个epoch的学习率：%f\" % (epoch+1,optimizer.param_groups[0]['lr']))\n    scheduler.step()\n    # The average loss and accuracy of the training set is the average of the recorded values.\n    train_loss = np.sum(train_losses) / len(train_losses)\n    # train_acc = np.sum(train_accs) / len(train_accs)\n    # Print the information.\n    # print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}\")\n    print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}\")\n\n  # Train process (all epochs) is complete\n  print('Training process has finished. Saving trained model.')\n  print('Starting validation')\n\n  # Saving the model\n  print('saving model with loss {:.3f}'.format(train_loss))\n  save_path = f'./model-fold-{fold}.pth'\n  torch.save(model.state_dict(),save_path)\n  # Start Validation\n  model.eval()\n  valid_losses = []\n  valid_accs = []\n  with torch.no_grad():\n    for batch in tqdm(validloader):\n      imgs, labels = batch\n      # No gradient in validation\n      logits = model(imgs.to(device))\n      loss = valid_loss_function(logits,labels.to(device))\n      acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n      # Record loss and accuracy\n      valid_losses.append(loss.item())        \n      valid_accs.append(acc)\n    # The average loss and accuracy\n    valid_loss = np.sum(valid_losses)/len(valid_losses)\n    valid_acc = np.sum(valid_accs)/len(valid_accs)\n    print(f\"[ Valid | {epoch + 1:03d}/{num_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}\")\n    print('Accuracy for fold %d: %d' % (fold, valid_acc))\n    print('--------------------------------------')\n    results[fold] = valid_acc\n# Print fold results\nprint(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')\nprint('--------------------------------')\ntotal_summation = 0.0\nfor key, value in results.items():\n  print(f'Fold {key}: {value} ')\n  total_summation += value\nprint(f'Average: {total_summation/len(results.items())} ')","metadata":{"id":"lwVkf8zm8z1W","outputId":"629b6575-6f85-4da3-a34d-b580b0172e1a","execution":{"iopub.status.busy":"2021-06-28T04:03:23.678839Z","iopub.status.idle":"2021-06-28T04:03:23.679278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **预测**","metadata":{"id":"4eQyBuNbOHzy"}},{"cell_type":"code","source":"testloader = torch.utils.data.DataLoader(\n                      TestData(test_path, img_path, transform = val_test_transform),\n                      batch_size=32, num_workers=0)","metadata":{"id":"aoUIdMxGXNk-","execution":{"iopub.status.busy":"2021-06-28T04:12:37.026383Z","iopub.execute_input":"2021-06-28T04:12:37.026763Z","iopub.status.idle":"2021-06-28T04:12:37.045990Z","shell.execute_reply.started":"2021-06-28T04:12:37.026724Z","shell.execute_reply":"2021-06-28T04:12:37.044640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## predict\nmodel = resnest_model(176)\n\n# create model and load weights from checkpoint\nmodel = model.to(device)\n# load the all folds\nfor test_fold in range(k_folds):\n  model_path = f'./model-fold-{test_fold}.pth'\n  saveFileName = f'./submission-fold-{test_fold}.csv'\n  model.load_state_dict(torch.load(model_path))\n\n  # Make sure the model is in eval mode.\n  # Some modules like Dropout or BatchNorm affect if the model is in training mode.\n  model.eval()\n  tta_model = tta.ClassificationTTAWrapper(model, tta.aliases.five_crop_transform(200,200)) # Test-Time Augmentation\n\n  # Initialize a list to store the predictions.\n  predictions = []\n  # Iterate the testing set by batches.\n  for batch in tqdm(testloader):\n      \n      imgs = batch\n      with torch.no_grad():\n          logits = tta_model(imgs.to(device))\n      \n      # Take the class with greatest logit as prediction and record it.\n      predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())\n\n  preds = []\n  for i in predictions:\n      preds.append(num_to_class[i])\n\n  test_data = pd.read_csv(test_path)\n  test_data['label'] = pd.Series(preds)\n  submission = pd.concat([test_data['image'], test_data['label']], axis=1)\n  submission.to_csv(saveFileName, index=False)\n  print(\"ResNeSt Model Results Done!!!!!!!!!!!!!!!!!!!!!!!!!!!\")","metadata":{"id":"0l-USFxpOYty","execution":{"iopub.status.busy":"2021-06-28T04:03:23.681531Z","iopub.status.idle":"2021-06-28T04:03:23.681939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **ResNeSt的5折交叉验证的结果投票**","metadata":{"id":"IjIr3i27APLP"}},{"cell_type":"code","source":"# 读取5折交叉验证的结果\ndf0 = pd.read_csv('./submission-fold-0.csv')\ndf1 = pd.read_csv('./submission-fold-1.csv')\ndf2 = pd.read_csv('./submission-fold-2.csv')\ndf3 = pd.read_csv('./submission-fold-3.csv')\ndf4 = pd.read_csv('./submission-fold-4.csv')","metadata":{"id":"PKieA1DR7j1d","execution":{"iopub.status.busy":"2021-06-28T04:03:23.682816Z","iopub.status.idle":"2021-06-28T04:03:23.683228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 往第0折结果里添加数字化标签列\nlist_num_label0 = []\nfor i in df0['label']:\n  list_num_label0.append(class_to_num[i])\ndf0['num_label0']=list_num_label0\ndf0.head()","metadata":{"id":"HukZ6ikN8ODD","execution":{"iopub.status.busy":"2021-06-28T04:03:23.684119Z","iopub.status.idle":"2021-06-28T04:03:23.684561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 往第1折结果里添加数字化标签列\nlist_num_label1 = []\nfor i in df1['label']:\n  list_num_label1.append(class_to_num[i])\ndf1['num_label1']=list_num_label1\ndf1.head()","metadata":{"id":"fVisaaKR8Pfq","execution":{"iopub.status.busy":"2021-06-28T04:03:23.685348Z","iopub.status.idle":"2021-06-28T04:03:23.685796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 往第2折结果里添加数字化标签列\nlist_num_label2 = []\nfor i in df2['label']:\n  list_num_label2.append(class_to_num[i])\ndf2['num_label2']=list_num_label2\ndf2.head()","metadata":{"id":"lUtmlGbU9E0x","execution":{"iopub.status.busy":"2021-06-28T04:03:23.686639Z","iopub.status.idle":"2021-06-28T04:03:23.687064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 往第3折结果里添加数字化标签列\nlist_num_label3 = []\nfor i in df3['label']:\n  list_num_label3.append(class_to_num[i])\ndf3['num_label3']=list_num_label3\ndf3.head()","metadata":{"id":"Coqj8NMf9KJq","execution":{"iopub.status.busy":"2021-06-28T04:03:23.687907Z","iopub.status.idle":"2021-06-28T04:03:23.688334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 往第4折结果里添加数字化标签列\nlist_num_label4 = []\nfor i in df4['label']:\n  list_num_label4.append(class_to_num[i])\ndf4['num_label4']=list_num_label4\ndf4.head()","metadata":{"id":"htPczSqT9Pua","execution":{"iopub.status.busy":"2021-06-28T04:03:23.68924Z","iopub.status.idle":"2021-06-28T04:03:23.689701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 准备整合5折的结果到同一个DataFrame\ndf_all = df0.copy()\ndf_all.drop(['label'],axis=1,inplace=True)\ndf_all.head()","metadata":{"id":"pt_MGztN9UJq","execution":{"iopub.status.busy":"2021-06-28T04:03:23.690722Z","iopub.status.idle":"2021-06-28T04:03:23.691148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 整合5折的数字化标签结果到同一个DataFrame\ndf_all['num_label1']=list_num_label1\ndf_all['num_label2']=list_num_label2\ndf_all['num_label3']=list_num_label3\ndf_all['num_label4']=list_num_label4\ndf_all.head()","metadata":{"id":"sOPq5bby-A76","execution":{"iopub.status.busy":"2021-06-28T04:03:23.692043Z","iopub.status.idle":"2021-06-28T04:03:23.692482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 对df_all进行转置，方便求众数\ndf_all_transpose = df_all.copy().drop(['image'],axis=1).transpose()\ndf_all_transpose.head()","metadata":{"id":"aLcatNuh-QKa","execution":{"iopub.status.busy":"2021-06-28T04:03:23.693289Z","iopub.status.idle":"2021-06-28T04:03:23.693721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 求得投票众数\ndf_mode = df_all_transpose.mode().transpose()\ndf_mode.head()","metadata":{"id":"4uo7hDjJ-dP6","execution":{"iopub.status.busy":"2021-06-28T04:03:23.694679Z","iopub.status.idle":"2021-06-28T04:03:23.695118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 把投票结果的数字化标签转成字符串标签\nvoting_class = []\nfor each in df_mode[0]:\n  voting_class.append(num_to_class[each])\nvoting_class","metadata":{"id":"o84YL1Sn-8qL","execution":{"iopub.status.busy":"2021-06-28T04:03:23.696006Z","iopub.status.idle":"2021-06-28T04:03:23.696423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 将投票结果的字符串标签添加到df_all中\ndf_all['label'] = voting_class\ndf_all.head()","metadata":{"id":"oU7E6R6H_nq4","execution":{"iopub.status.busy":"2021-06-28T04:03:23.697296Z","iopub.status.idle":"2021-06-28T04:03:23.697727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 提取image和label两列为最终的结果\ndf_submission = df_all[['image','label']].copy()\ndf_submission.head()","metadata":{"id":"RBL_QsyY_1go","execution":{"iopub.status.busy":"2021-06-28T04:03:23.698635Z","iopub.status.idle":"2021-06-28T04:03:23.699058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 保存当前模型得到的最终结果\ndf_submission.to_csv('./submission-resnest.csv', index=False)\nprint('Voting results of resnest successfully saved!')","metadata":{"id":"LfhAzq4yAGSI","execution":{"iopub.status.busy":"2021-06-28T04:03:23.700112Z","iopub.status.idle":"2021-06-28T04:03:23.700645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **基于ResNeXt模型部分**","metadata":{}},{"cell_type":"markdown","source":"### **ResNeXt模型**","metadata":{"id":"nk1vuWbh-7Ry"}},{"cell_type":"code","source":"# 是否要冻住模型的前面一些层\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        model = model\n        for param in model.parameters():\n            param.requires_grad = False\n\n# resnext50_32x4d模型\ndef resnext_model(num_classes, feature_extract = False, use_pretrained=True):\n\n    model_ft = models.resnext50_32x4d(pretrained=use_pretrained)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))\n\n    return model_ft","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.701525Z","iopub.status.idle":"2021-06-28T04:03:23.701969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration options\nk_folds = 5\nnum_epochs = 30\nlearning_rate = 1e-3\nweight_decay = 1e-3\ntrain_loss_function = CutMixCrossEntropyLoss(True)\nvalid_loss_function = nn.CrossEntropyLoss()\n# For fold results\nresults = {}\n\n# Set fixed random number seed\ntorch.manual_seed(42)\n\n# Define the K-fold Cross Validator\nkfold = KFold(n_splits=k_folds, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.702859Z","iopub.status.idle":"2021-06-28T04:03:23.703278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **训练**","metadata":{}},{"cell_type":"code","source":"# Start print\nprint('--------------------------------------')\n\n# K-fold Cross Validation model evaluation\nfor fold, (train_ids,valid_ids) in enumerate(kfold.split(train_val_dataset)):\n\n  # Print\n  print(f'FOLD {fold}')\n  print('--------------------------------------')\n\n  # Sample elements randomly from a given list of ids, no replacement.\n  train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n  valid_subsampler = torch.utils.data.SubsetRandomSampler(valid_ids)\n\n  # Define data loaders for training and testing data in this fold\n  trainloader = torch.utils.data.DataLoader(\n                      CutMix(TrainValidData(train_val_path, img_path, transform = train_transform), num_class=176, beta=1.0, prob=0.5, num_mix=2), \n                      batch_size=128, sampler=train_subsampler, num_workers=0)\n  validloader = torch.utils.data.DataLoader(\n                      TrainValidData(train_val_path, img_path, transform = val_test_transform),\n                      batch_size=128, sampler=valid_subsampler, num_workers=0)\n  \n  # Initialize a model and put it on the device specified.\n  model = resnext_model(176)\n  model = model.to(device)\n  model.device = device\n  \n  # Initialize optimizer\n  optimizer = torch.optim.AdamW(model.parameters(),lr=learning_rate,weight_decay= weight_decay)\n#   optimizer = SWA(our_optimizer, swa_start=5, swa_freq =5, swa_lr=0.05)\n  scheduler = CosineAnnealingLR(optimizer,T_max=10)\n\n  # Run the training loop for defined number of epochs\n  for epoch in range(0,num_epochs):\n    model.train()\n    # Print epoch\n    print(f'Starting epoch {epoch+1}')\n    # These are used to record information in training\n    train_losses = []\n    train_accs = []\n    # Iterate the training set by batches\n    for batch in tqdm(trainloader):\n      # Move images and labels to GPU\n      imgs, labels = batch\n      imgs = imgs.to(device)\n      labels = labels.to(device)\n      # Forward the data\n      logits = model(imgs)\n      # Calculate loss\n      loss = train_loss_function(logits,labels)\n      # Clear gradients in previous step\n      optimizer.zero_grad()\n      # Compute gradients for parameters\n      loss.backward()\n      # Update the parameters with computed gradients\n      optimizer.step()\n      # Compute the accuracy for current batch.\n      # acc = (logits.argmax(dim=-1) == labels).float().mean()\n      # Record the loss and accuracy.\n      train_losses.append(loss.item())\n      # train_accs.append(acc)\n    print(\"第%d个epoch的学习率：%f\" % (epoch+1,optimizer.param_groups[0]['lr']))\n    scheduler.step()\n    # The average loss and accuracy of the training set is the average of the recorded values.\n    train_loss = np.sum(train_losses) / len(train_losses)\n    # train_acc = np.sum(train_accs) / len(train_accs)\n    # Print the information.\n    # print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}\")\n    print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}\")\n\n  # Train process (all epochs) is complete\n  print('Training process has finished. Saving trained model.')\n  print('Starting validation')\n\n  # Saving the model\n  print('saving model with loss {:.3f}'.format(train_loss))\n  save_path = f'./model-fold-{fold}.pth'\n  torch.save(model.state_dict(),save_path)\n  # Start Validation\n  model.eval()\n  valid_losses = []\n  valid_accs = []\n  with torch.no_grad():\n    for batch in tqdm(validloader):\n      imgs, labels = batch\n      # No gradient in validation\n      logits = model(imgs.to(device))\n      loss = valid_loss_function(logits,labels.to(device))\n      acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n      # Record loss and accuracy\n      valid_losses.append(loss.item())        \n      valid_accs.append(acc)\n    # The average loss and accuracy\n    valid_loss = np.sum(valid_losses)/len(valid_losses)\n    valid_acc = np.sum(valid_accs)/len(valid_accs)\n    print(f\"[ Valid | {epoch + 1:03d}/{num_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}\")\n    print('Accuracy for fold %d: %d' % (fold, valid_acc))\n    print('--------------------------------------')\n    results[fold] = valid_acc\n# Print fold results\nprint(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')\nprint('--------------------------------')\ntotal_summation = 0.0\nfor key, value in results.items():\n  print(f'Fold {key}: {value} ')\n  total_summation += value\nprint(f'Average: {total_summation/len(results.items())} ')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.704534Z","iopub.status.idle":"2021-06-28T04:03:23.704982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **预测**","metadata":{}},{"cell_type":"code","source":"testloader = torch.utils.data.DataLoader(\n                      TestData(test_path, img_path, transform = val_test_transform),\n                      batch_size=128, num_workers=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.70594Z","iopub.status.idle":"2021-06-28T04:03:23.706365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## predict\nmodel = resnext_model(176)\n\n# create model and load weights from checkpoint\nmodel = model.to(device)\n# load the all folds\nfor test_fold in range(k_folds):\n  model_path = f'./model-fold-{test_fold}.pth'\n  saveFileName = f'./submission-fold-{test_fold}.csv'\n  model.load_state_dict(torch.load(model_path))\n\n  # Make sure the model is in eval mode.\n  # Some modules like Dropout or BatchNorm affect if the model is in training mode.\n  model.eval()\n  tta_model = tta.ClassificationTTAWrapper(model, tta.aliases.five_crop_transform(200,200))\n\n  # Initialize a list to store the predictions.\n  predictions = []\n  # Iterate the testing set by batches.\n  for batch in tqdm(testloader):\n      \n      imgs = batch\n      with torch.no_grad():\n          logits = tta_model(imgs.to(device))\n      \n      # Take the class with greatest logit as prediction and record it.\n      predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())\n\n  preds = []\n  for i in predictions:\n      preds.append(num_to_class[i])\n\n  test_data = pd.read_csv(test_path)\n  test_data['label'] = pd.Series(preds)\n  submission = pd.concat([test_data['image'], test_data['label']], axis=1)\n  submission.to_csv(saveFileName, index=False)\n  print(\"ResNeXt Model Results Done!!!!!!!!!!!!!!!!!!!!!!!!!!!\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.707306Z","iopub.status.idle":"2021-06-28T04:03:23.707748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **ResNeXt的5折交叉验证的结果投票**","metadata":{}},{"cell_type":"code","source":"df0 = pd.read_csv('./submission-fold-0.csv')\ndf1 = pd.read_csv('./submission-fold-1.csv')\ndf2 = pd.read_csv('./submission-fold-2.csv')\ndf3 = pd.read_csv('./submission-fold-3.csv')\ndf4 = pd.read_csv('./submission-fold-4.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.70882Z","iopub.status.idle":"2021-06-28T04:03:23.709244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label0 = []\nfor i in df0['label']:\n  list_num_label0.append(class_to_num[i])\ndf0['num_label0']=list_num_label0\ndf0.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.710194Z","iopub.status.idle":"2021-06-28T04:03:23.71064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label1 = []\nfor i in df1['label']:\n  list_num_label1.append(class_to_num[i])\ndf1['num_label1']=list_num_label1\ndf1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.711472Z","iopub.status.idle":"2021-06-28T04:03:23.711902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label2 = []\nfor i in df2['label']:\n  list_num_label2.append(class_to_num[i])\ndf2['num_label2']=list_num_label2\ndf2.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.712976Z","iopub.status.idle":"2021-06-28T04:03:23.713435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label3 = []\nfor i in df3['label']:\n  list_num_label3.append(class_to_num[i])\ndf3['num_label3']=list_num_label3\ndf3.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.714354Z","iopub.status.idle":"2021-06-28T04:03:23.714793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label4 = []\nfor i in df4['label']:\n  list_num_label4.append(class_to_num[i])\ndf4['num_label4']=list_num_label4\ndf4.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.715627Z","iopub.status.idle":"2021-06-28T04:03:23.716073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = df0.copy()\ndf_all.drop(['label'],axis=1,inplace=True)\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.717018Z","iopub.status.idle":"2021-06-28T04:03:23.717445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['num_label1']=list_num_label1\ndf_all['num_label2']=list_num_label2\ndf_all['num_label3']=list_num_label3\ndf_all['num_label4']=list_num_label4\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.718335Z","iopub.status.idle":"2021-06-28T04:03:23.718781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_transpose = df_all.copy().drop(['image'],axis=1).transpose()\ndf_all_transpose.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.719616Z","iopub.status.idle":"2021-06-28T04:03:23.720043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mode = df_all_transpose.mode().transpose()\ndf_mode.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.721088Z","iopub.status.idle":"2021-06-28T04:03:23.72152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_class = []\nfor each in df_mode[0]:\n  voting_class.append(num_to_class[each])\nvoting_class","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.722314Z","iopub.status.idle":"2021-06-28T04:03:23.722732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['label'] = voting_class\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.723674Z","iopub.status.idle":"2021-06-28T04:03:23.724087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = df_all[['image','label']].copy()\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.725061Z","iopub.status.idle":"2021-06-28T04:03:23.72551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('./submission-resnext.csv', index=False)\nprint('ResNeXt voting results successfully saved!')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.726404Z","iopub.status.idle":"2021-06-28T04:03:23.72684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **基于DenseNet模型部分**","metadata":{}},{"cell_type":"markdown","source":"### **DenseNet模型**","metadata":{}},{"cell_type":"code","source":"# 是否要冻住模型的前面一些层\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        model = model\n        for param in model.parameters():\n            param.requires_grad = False\n\n# densenet161模型\ndef dense_model(num_classes, feature_extract = False, use_pretrained=True):\n\n    model_ft = models.densenet161(pretrained=use_pretrained)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    num_ftrs = model_ft.classifier.in_features\n    model_ft.classifier = nn.Sequential(nn.Linear(num_ftrs, num_classes))\n\n    return model_ft","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.727889Z","iopub.status.idle":"2021-06-28T04:03:23.728325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration options\nk_folds = 5\nnum_epochs = 30\nlearning_rate = 1e-4\nweight_decay = 1e-3\ntrain_loss_function = CutMixCrossEntropyLoss(True)\nvalid_loss_function = nn.CrossEntropyLoss()\n# For fold results\nresults = {}\n\n# Set fixed random number seed\ntorch.manual_seed(42)\n\n# Define the K-fold Cross Validator\nkfold = KFold(n_splits=k_folds, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.729239Z","iopub.status.idle":"2021-06-28T04:03:23.729685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **训练**","metadata":{}},{"cell_type":"code","source":"# Start print\nprint('--------------------------------------')\n\n# K-fold Cross Validation model evaluation\nfor fold, (train_ids,valid_ids) in enumerate(kfold.split(train_val_dataset)):\n\n  # Print\n  print(f'FOLD {fold}')\n  print('--------------------------------------')\n\n  # Sample elements randomly from a given list of ids, no replacement.\n  train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n  valid_subsampler = torch.utils.data.SubsetRandomSampler(valid_ids)\n\n  # Define data loaders for training and testing data in this fold\n  trainloader = torch.utils.data.DataLoader(\n                      CutMix(TrainValidData(train_val_path, img_path, transform = train_transform), num_class=176, beta=1.0, prob=0.5, num_mix=2), \n                      batch_size=32, sampler=train_subsampler, num_workers=0)\n  validloader = torch.utils.data.DataLoader(\n                      TrainValidData(train_val_path, img_path, transform = val_test_transform),\n                      batch_size=32, sampler=valid_subsampler, num_workers=0)\n  \n  # Initialize a model and put it on the device specified.\n  model = dense_model(176)\n  model = model.to(device)\n  model.device = device\n  \n  # Initialize optimizer\n  optimizer = torch.optim.AdamW(model.parameters(),lr=learning_rate,weight_decay= weight_decay)\n  scheduler = CosineAnnealingLR(optimizer,T_max=10)\n\n  # Run the training loop for defined number of epochs\n  for epoch in range(0,num_epochs):\n    model.train()\n    # Print epoch\n    print(f'Starting epoch {epoch+1}')\n    # These are used to record information in training\n    train_losses = []\n    train_accs = []\n    # Iterate the training set by batches\n    for batch in tqdm(trainloader):\n      # Move images and labels to GPU\n      imgs, labels = batch\n      imgs = imgs.to(device)\n      labels = labels.to(device)\n      # Forward the data\n      logits = model(imgs)\n      # Calculate loss\n      loss = train_loss_function(logits,labels)\n      # Clear gradients in previous step\n      optimizer.zero_grad()\n      # Compute gradients for parameters\n      loss.backward()\n      # Update the parameters with computed gradients\n      optimizer.step()\n      # Compute the accuracy for current batch.\n#       acc = (logits.argmax(dim=-1) == labels).float().mean()\n      # Record the loss and accuracy.\n      train_losses.append(loss.item())\n#       train_accs.append(acc)\n    print(\"第%d个epoch的学习率：%f\" % (epoch+1,optimizer.param_groups[0]['lr']))\n    scheduler.step()\n    # The average loss and accuracy of the training set is the average of the recorded values.\n    train_loss = np.sum(train_losses) / len(train_losses)\n#     train_acc = np.sum(train_accs) / len(train_accs)\n    # Print the information.\n#     print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}\")\n    print(f\"[ Train | {epoch + 1:03d}/{num_epochs:03d} ] loss = {train_loss:.5f}\")\n\n  # Train process (all epochs) is complete\n  print('Training process has finished. Saving trained model.')\n  print('Starting validation')\n\n  # Saving the model\n  print('saving model with loss {:.3f}'.format(train_loss))\n  save_path = f'./model-fold-{fold}.pth'\n  torch.save(model.state_dict(),save_path)\n  # Start Validation\n  model.eval()\n  valid_losses = []\n  valid_accs = []\n  with torch.no_grad():\n    for batch in tqdm(validloader):\n      imgs, labels = batch\n      # No gradient in validation\n      logits = model(imgs.to(device))\n      loss = valid_loss_function(logits,labels.to(device))\n      acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n      # Record loss and accuracy\n      valid_losses.append(loss.item())        \n      valid_accs.append(acc)\n    # The average loss and accuracy\n    valid_loss = np.sum(valid_losses)/len(valid_losses)\n    valid_acc = np.sum(valid_accs)/len(valid_accs)\n    print(f\"[ Valid | {epoch + 1:03d}/{num_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}\")\n    print('Accuracy for fold %d: %d' % (fold, valid_acc))\n    print('--------------------------------------')\n    results[fold] = valid_acc\n# Print fold results\nprint(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')\nprint('--------------------------------')\ntotal_summation = 0.0\nfor key, value in results.items():\n  print(f'Fold {key}: {value} ')\n  total_summation += value\nprint(f'Average: {total_summation/len(results.items())} ')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.730947Z","iopub.status.idle":"2021-06-28T04:03:23.731377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **预测**","metadata":{}},{"cell_type":"code","source":"testloader = torch.utils.data.DataLoader(\n                      TestData(test_path, img_path, transform = val_test_transform),\n                      batch_size=32, num_workers=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.732239Z","iopub.status.idle":"2021-06-28T04:03:23.732679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## predict\nmodel = dense_model(176)\n\n# create model and load weights from checkpoint\nmodel = model.to(device)\n# load the all folds\nfor test_fold in range(k_folds):\n  model_path = f'./model-fold-{test_fold}.pth'\n  saveFileName = f'./submission-fold-{test_fold}.csv'\n  model.load_state_dict(torch.load(model_path))\n\n  # Make sure the model is in eval mode.\n  # Some modules like Dropout or BatchNorm affect if the model is in training mode.\n  model.eval()\n  tta_model = tta.ClassificationTTAWrapper(model, tta.aliases.five_crop_transform(200,200))\n\n  # Initialize a list to store the predictions.\n  predictions = []\n  # Iterate the testing set by batches.\n  for batch in tqdm(testloader):\n      \n      imgs = batch\n      with torch.no_grad():\n          logits = tta_model(imgs.to(device))\n      \n      # Take the class with greatest logit as prediction and record it.\n      predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())\n\n  preds = []\n  for i in predictions:\n      preds.append(num_to_class[i])\n\n  test_data = pd.read_csv(test_path)\n  test_data['label'] = pd.Series(preds)\n  submission = pd.concat([test_data['image'], test_data['label']], axis=1)\n  submission.to_csv(saveFileName, index=False)\n  print(\"Dense Model Results Done!!!!!!!!!!!!!!!!!!!!!!!!!!!\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.733527Z","iopub.status.idle":"2021-06-28T04:03:23.733952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **DenseNet的5折交叉验证的结果投票**","metadata":{}},{"cell_type":"code","source":"df0 = pd.read_csv('./submission-fold-0.csv')\ndf1 = pd.read_csv('./submission-fold-1.csv')\ndf2 = pd.read_csv('./submission-fold-2.csv')\ndf3 = pd.read_csv('./submission-fold-3.csv')\ndf4 = pd.read_csv('./submission-fold-4.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.734843Z","iopub.status.idle":"2021-06-28T04:03:23.735287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label0 = []\nfor i in df0['label']:\n  list_num_label0.append(class_to_num[i])\ndf0['num_label0']=list_num_label0\ndf0.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.736175Z","iopub.status.idle":"2021-06-28T04:03:23.73661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label1 = []\nfor i in df1['label']:\n  list_num_label1.append(class_to_num[i])\ndf1['num_label1']=list_num_label1\ndf1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.737544Z","iopub.status.idle":"2021-06-28T04:03:23.73797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label2 = []\nfor i in df2['label']:\n  list_num_label2.append(class_to_num[i])\ndf2['num_label2']=list_num_label2\ndf2.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.738824Z","iopub.status.idle":"2021-06-28T04:03:23.739257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label3 = []\nfor i in df3['label']:\n  list_num_label3.append(class_to_num[i])\ndf3['num_label3']=list_num_label3\ndf3.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.740182Z","iopub.status.idle":"2021-06-28T04:03:23.740626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_num_label4 = []\nfor i in df4['label']:\n  list_num_label4.append(class_to_num[i])\ndf4['num_label4']=list_num_label4\ndf4.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.741603Z","iopub.status.idle":"2021-06-28T04:03:23.742039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = df0.copy()\ndf_all.drop(['label'],axis=1,inplace=True)\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.742948Z","iopub.status.idle":"2021-06-28T04:03:23.743388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['num_label1']=list_num_label1\ndf_all['num_label2']=list_num_label2\ndf_all['num_label3']=list_num_label3\ndf_all['num_label4']=list_num_label4\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.744319Z","iopub.status.idle":"2021-06-28T04:03:23.744771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_transpose = df_all.copy().drop(['image'],axis=1).transpose()\ndf_all_transpose.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.745659Z","iopub.status.idle":"2021-06-28T04:03:23.746088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mode = df_all_transpose.mode().transpose()\ndf_mode.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.74706Z","iopub.status.idle":"2021-06-28T04:03:23.747458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_class = []\nfor each in df_mode[0]:\n  voting_class.append(num_to_class[each])\nvoting_class","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.748325Z","iopub.status.idle":"2021-06-28T04:03:23.748741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['label'] = voting_class\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.749734Z","iopub.status.idle":"2021-06-28T04:03:23.750197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = df_all[['image','label']].copy()\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.751176Z","iopub.status.idle":"2021-06-28T04:03:23.751607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('./submission-densenet.csv', index=False)\nprint('Densenet results successfully saved!')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:03:23.752542Z","iopub.status.idle":"2021-06-28T04:03:23.752961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **最终结果集成（投票方式）**","metadata":{}},{"cell_type":"code","source":"df_resnest = pd.read_csv('../input/classify-leaves-results/submission-resnest.csv')\ndf_resnext = pd.read_csv('../input/classify-leaves-results/submission-resnext.csv')\ndf_densenet = pd.read_csv('../input/classify-leaves-results/submission-densenet.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:13:53.153683Z","iopub.execute_input":"2021-06-28T04:13:53.154162Z","iopub.status.idle":"2021-06-28T04:13:53.213985Z","shell.execute_reply.started":"2021-06-28T04:13:53.154127Z","shell.execute_reply":"2021-06-28T04:13:53.212754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = df_resnest.copy()\ndf_all.rename(columns = {'label':'label_resnest'},inplace=True)\ndf_all['label_resnext'] = df_resnext.copy()['label']\ndf_all['label_densenet'] = df_densenet.copy()['label']\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:13:55.512212Z","iopub.execute_input":"2021-06-28T04:13:55.512599Z","iopub.status.idle":"2021-06-28T04:13:55.529233Z","shell.execute_reply.started":"2021-06-28T04:13:55.512565Z","shell.execute_reply":"2021-06-28T04:13:55.528516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['label']=0\nfor rows in range(len(df_all)):\n    if (df_all['label_resnest'].iloc[rows]==df_all['label_resnext'].iloc[rows]) or (df_all['label_resnest'].iloc[rows]==df_all['label_densenet'].iloc[rows]):\n        df_all['label'].iloc[rows] = df_all.copy()['label_resnest'].iloc[rows]\n    elif df_all['label_resnext'].iloc[rows]==df_all['label_densenet'].iloc[rows]:\n        df_all['label'].iloc[rows] = df_all.copy()['label_resnext'].iloc[rows]\n    else:\n        df_all['label'].iloc[rows] = df_all.copy()['label_resnest'].iloc[rows]\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:13:59.038907Z","iopub.execute_input":"2021-06-28T04:13:59.039433Z","iopub.status.idle":"2021-06-28T04:14:08.864030Z","shell.execute_reply.started":"2021-06-28T04:13:59.039400Z","shell.execute_reply":"2021-06-28T04:14:08.862979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = df_all.copy()[['image','label']]\ndf_final.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:14:13.240869Z","iopub.execute_input":"2021-06-28T04:14:13.241434Z","iopub.status.idle":"2021-06-28T04:14:13.256092Z","shell.execute_reply.started":"2021-06-28T04:14:13.241384Z","shell.execute_reply":"2021-06-28T04:14:13.254427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.to_csv('./submission.csv', index=False)\nprint('Final results successfully saved!')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:14:17.970596Z","iopub.execute_input":"2021-06-28T04:14:17.970960Z","iopub.status.idle":"2021-06-28T04:14:18.005325Z","shell.execute_reply.started":"2021-06-28T04:14:17.970929Z","shell.execute_reply":"2021-06-28T04:14:18.004214Z"},"trusted":true},"execution_count":null,"outputs":[]}]}