{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 首先导入包\n! pip install ipdb\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom PIL import Image\nimport os\n\nimport matplotlib.pyplot as plt\nimport torchvision.models as models\nfrom torchvision.transforms.transforms import RandomCrop\n# This is for the progress bar.\nfrom tqdm import tqdm\nfrom torch.utils.tensorboard import SummaryWriter\nimport ipdb\nimport albumentations\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-28T16:09:54.866064Z","iopub.execute_input":"2021-06-28T16:09:54.866406Z","iopub.status.idle":"2021-06-28T16:10:07.768073Z","shell.execute_reply.started":"2021-06-28T16:09:54.866374Z","shell.execute_reply":"2021-06-28T16:10:07.767265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#CutMix方法\n\"\"\"输入为：样本的size和生成的随机lamda值\"\"\"\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    \"\"\"1.论文里的公式2，求出B的rw,rh\"\"\"\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n \n    # uniform\n    \"\"\"2.论文里的公式2，求出B的rx,ry（bbox的中心点）\"\"\"\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    #限制坐标区域不超过样本大小\n \n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    \"\"\"3.返回剪裁B区域的坐标值\"\"\"\n    return bbx1, bby1, bbx2, bby2\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:07.769615Z","iopub.execute_input":"2021-06-28T16:10:07.769967Z","iopub.status.idle":"2021-06-28T16:10:07.778249Z","shell.execute_reply.started":"2021-06-28T16:10:07.769931Z","shell.execute_reply":"2021-06-28T16:10:07.777349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_device():\n    return 'cuda' if torch.cuda.is_available() else 'cpu'\n\ndevice = get_device()\ntorch.backends.cudnn.benchmark = True\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:10.127805Z","iopub.execute_input":"2021-06-28T16:10:10.128138Z","iopub.status.idle":"2021-06-28T16:10:10.133185Z","shell.execute_reply.started":"2021-06-28T16:10:10.128108Z","shell.execute_reply":"2021-06-28T16:10:10.132332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#导入数据\nlabels_dataframe = pd.read_csv('../input/classify-leaves/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:15.18289Z","iopub.execute_input":"2021-06-28T16:10:15.183348Z","iopub.status.idle":"2021-06-28T16:10:15.234074Z","shell.execute_reply.started":"2021-06-28T16:10:15.183306Z","shell.execute_reply":"2021-06-28T16:10:15.23329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 把label文件排个序\nleaves_labels = sorted(list(set(labels_dataframe['label'])))\nn_classes = len(leaves_labels)\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nnum_to_class = {v : k for k, v in class_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:16.703443Z","iopub.execute_input":"2021-06-28T16:10:16.703767Z","iopub.status.idle":"2021-06-28T16:10:16.71486Z","shell.execute_reply.started":"2021-06-28T16:10:16.703737Z","shell.execute_reply":"2021-06-28T16:10:16.713942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass LeavesData(Dataset):\n    def __init__(self, csv_path, file_path, mode='train', valid_ratio=0.2, resize_height=256, resize_width=256):\n        \"\"\"\n        Args:\n            csv_path (string): csv 文件路径\n            img_path (string): 图像文件所在路径\n            mode (string): 训练模式还是测试模式\n            valid_ratio (float): 验证集比例\n        \"\"\"\n        \n        # 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#\n        self.resize_height = resize_height\n        self.resize_width = resize_width\n\n        self.file_path = file_path\n        self.mode = mode\n\n        # 读取 csv 文件\n        # 利用pandas读取csv文件\n        self.data_info = pd.read_csv(csv_path)  #header=None是去掉表头部分\n        self.data_info = self.data_info.sample(frac=1.0)\n        # 计算 length\n        self.data_len = len(self.data_info.index)\n        self.train_len = int(self.data_len * (1 - valid_ratio))\n        \n        if mode == 'train':\n            # 第一列包含图像文件的名称\n            # self.train_image = np.asarray(self.data_info.iloc[1:, 0])\n            self.train_image = np.asarray(self.data_info.iloc[0:self.train_len, 0])  #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始到train_len\n            # 第二列是图像的 label\n            # self.train_label = np.asarray(self.data_info.iloc[1:, 1])\n            self.train_label = np.asarray(self.data_info.iloc[0:self.train_len, 1])\n\n            self.image_arr = self.train_image \n            self.label_arr = self.train_label\n        elif mode == 'valid':\n            self.valid_image = np.asarray(self.data_info.iloc[self.train_len:, 0])  \n            self.valid_label = np.asarray(self.data_info.iloc[self.train_len:, 1])\n            self.image_arr = self.valid_image\n            self.label_arr = self.valid_label\n        elif mode == 'test':\n            self.test_image = np.asarray(self.data_info.iloc[1:, 0])\n            self.image_arr = self.test_image\n            \n        self.real_len = len(self.image_arr)\n\n        print('Finished reading the {} set of Leaves Dataset ({} samples found)'\n              .format(mode, self.real_len))\n\n    def __getitem__(self, index):\n        # 从 image_arr中得到索引对应的文件名\n        single_image_name = self.image_arr[index]\n\n        # 读取图像文件\n        img_as_img = Image.open(self.file_path + single_image_name)\n\n        #如果需要将RGB三通道的图片转换成灰度图片可参考下面两行\n#         if img_as_img.mode != 'L':\n#             img_as_img = img_as_img.convert('L')\n\n        #设置好需要转换的变量，还可以包括一系列的nomarlize等等操作\n        if self.mode == 'train':\n            transform = transforms.Compose([\n                # transforms.CenterCrop(224),\n                # transforms.TenCrop(224, vertical_flip=False),\n                transforms.Resize((224, 224)),\n                transforms.CenterCrop(size=224),\n                # transforms.TenCrop\n                # transforms.RandomCrop(224),\n                transforms.RandomVerticalFlip() ,\n                # transforms.RandomResizedCrop(size=224, scale=(0.8, 1.0)),\n                # transforms.RandomRotation(degrees=15),\n                transforms.RandomHorizontalFlip(),\n                #transforms.CenterCrop(size=224),\n                transforms.ToTensor(),\n                # transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n            ])\n\n\n\n        else:\n            # valid和test不做数据增强\n            transform = transforms.Compose([\n                transforms.Resize((224, 224)),\n                transforms.CenterCrop(size=224),\n                transforms.ToTensor(),\n                # transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n            ])\n        \n        img_as_img = transform(img_as_img)\n        \n        if self.mode == 'test':\n            return img_as_img\n        else:\n            # 得到图像的 string label\n            label = self.label_arr[index]\n            # number label\n            number_label = class_to_num[label]\n\n            return img_as_img, number_label  #返回每一个index对应的图片数据和对应的label\n\n    def __len__(self):\n        return self.real_len","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:21.637062Z","iopub.execute_input":"2021-06-28T16:10:21.637391Z","iopub.status.idle":"2021-06-28T16:10:21.651732Z","shell.execute_reply.started":"2021-06-28T16:10:21.637358Z","shell.execute_reply":"2021-06-28T16:10:21.650652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \n\ntrain_path = '../input/classify-leaves/train.csv'\ntest_path = '../input/classify-leaves/test.csv'\n# csv文件中已经images的路径了，因此这里只到上一级目录\nimg_path = '../input/classify-leaves/'\n\ntrain_dataset = LeavesData(train_path, img_path, mode='train')\nval_dataset = LeavesData(train_path, img_path, mode='valid')\ntest_dataset = LeavesData(test_path, img_path, mode='test')\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:25.559682Z","iopub.execute_input":"2021-06-28T16:10:25.559997Z","iopub.status.idle":"2021-06-28T16:10:25.616235Z","shell.execute_reply.started":"2021-06-28T16:10:25.559969Z","shell.execute_reply":"2021-06-28T16:10:25.615371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 定义data loader\ntrain_loader = torch.utils.data.DataLoader(\n        dataset=train_dataset,\n        batch_size=batch_size, \n        shuffle=True,\n        num_workers=8,\n        pin_memory=True\n    )\n\nval_loader = torch.utils.data.DataLoader(\n        dataset=val_dataset,\n        batch_size=batch_size, \n        shuffle=False,\n        num_workers=8,\n        pin_memory=True\n    )\ntest_loader = torch.utils.data.DataLoader(\n        dataset=test_dataset,\n        batch_size=batch_size, \n        shuffle=False,\n        num_workers=8,\n        pin_memory=True\n    )","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:34.50133Z","iopub.execute_input":"2021-06-28T16:10:34.501651Z","iopub.status.idle":"2021-06-28T16:10:34.508719Z","shell.execute_reply.started":"2021-06-28T16:10:34.501623Z","shell.execute_reply":"2021-06-28T16:10:34.507515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        model = model\n        for param in model.parameters():\n            param.requires_grad = False\n\n# resnext101_32x8d\ndef res_model(num_classes, feature_extract = False, use_pretrained=True):\n\n    model_ft = models.resnext101_32x8d(pretrained=use_pretrained)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))\n\n    return model_ft\n\n# 超参数\nlearning_rate = 3e-4\nweight_decay = 1e-3\nnum_epoch = 50\nbeta = 0.9\ncutmix_prob = 0.9\nmodel_path = './pre_res_model.ckpt'\n\n# torch.distributed.init_process_group(backend=\"nccl\")\nmodel = res_model(176)\nmodel = nn.DataParallel(model)\nmodel = model.to(device)\n# model = nn.parallel.DistributedDataParallel(model) # device_ids will include all GPU devices by default\n\n\n# For the classification task, we use cross-entropy as the measurement of performance.\ncriterion = nn.CrossEntropyLoss()\n\n# Initialize optimizer, you may fine-tune some hyperparameters such as learning rate on your own.\noptimizer = torch.optim.AdamW(model.parameters(), lr = learning_rate, weight_decay=0.02)\n# optimizer = torch.optim.SGD(model.parameters(),lr=learning_rate)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 10, eta_min=1e-6, last_epoch=-1)\n# The number of training epochs.\nn_epochs = num_epoch\n\n# ipdb.set_trace()\nbest_acc = 0.0","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:10:41.359205Z","iopub.execute_input":"2021-06-28T16:10:41.359568Z","iopub.status.idle":"2021-06-28T16:10:51.365268Z","shell.execute_reply.started":"2021-06-28T16:10:41.359536Z","shell.execute_reply":"2021-06-28T16:10:51.364415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(n_epochs):\n    # ---------- Training ----------\n    # Make sure the model is in train mode before training.\n    model.train() \n    # These are used to record information in training.\n    train_loss = []\n    train_accs = []\n    # Iterate the training set by batches.\n    for batch in tqdm(train_loader):\n        # A batch consists of image data and corresponding labels.\n        imgs, labels = batch\n        imgs = imgs.to(device)\n        labels = labels.to(device)\n        \n        r = np.random.rand(1)\n\n        if beta > 0 and r < cutmix_prob:\n            # generate mixed sample\n            \"\"\"1.设定lamda的值，服从beta分布\"\"\"\n            lam = np.random.beta(beta, beta)\n            \"\"\"2.找到两个随机样本\"\"\"\n            rand_index = torch.randperm(imgs.size()[0]).cuda()\n            target_a = labels#一个batch\n            target_b = labels[rand_index] #batch中的某一张\n            \"\"\"3.生成剪裁区域B\"\"\"\n            bbx1, bby1, bbx2, bby2 = rand_bbox(imgs.size(), lam)\n            \"\"\"4.将原有的样本A中的B区域，替换成样本B中的B区域\"\"\"\n            imgs[:, :, bbx1:bbx2, bby1:bby2] = imgs[rand_index, :, bbx1:bbx2, bby1:bby2]\n            # adjust lambda to exactly match pixel ratio\n            \"\"\"5.根据剪裁区域坐标框的值调整lam的值\"\"\"\n            lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (imgs.size()[-1] * imgs.size()[-2]))\n            # compute output\n            \"\"\"6.将生成的新的训练样本丢到模型中进行训练\"\"\"\n            logits = model(imgs)\n            \"\"\"7.按lamda值分配权重\"\"\"\n            loss = criterion(logits, target_a) * lam + criterion(logits, target_b) * (1. - lam)\n        else:\n                # compute output\n            # Forward the data. (Make sure data and model are on the same device.)\n            logits = model(imgs)\n            # Calculate the cross-entropy loss.\n            # We don't need to apply softmax before computing cross-entropy as it is done automatically.\n            loss = criterion(logits, labels)\n\n        \n        \n        # Gradients stored in the parameters in the previous step should be cleared out first.\n        optimizer.zero_grad()\n        # Compute the gradients for parameters.\n        loss.backward()\n        # Update the parameters with computed gradients.\n        optimizer.step()\n        \n        \n        # Compute the accuracy for current batch.\n        acc = (logits.argmax(dim=-1) == labels).float().mean()\n\n        # Record the loss and accuracy.\n        train_loss.append(loss.item())\n        train_accs.append(acc)\n        \n    scheduler.step()    \n    # The average loss and accuracy of the training set is the average of the recorded values.\n    train_loss = sum(train_loss) / len(train_loss)\n    train_acc = sum(train_accs) / len(train_accs)\n\n    # Print the information.\n    print(f\"[ Train | {epoch + 1:03d}/{n_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f},best_acc = {best_acc:.5f}\")\n    \n    \n    # ---------- Validation ----------\n    # Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.\n    model.eval()\n    # These are used to record information in validation.\n    valid_loss = []\n    valid_accs = []\n    \n    # Iterate the validation set by batches.\n    for batch in tqdm(val_loader):\n        imgs, labels = batch\n        # We don't need gradient in validation.\n        # Using torch.no_grad() accelerates the forward process.\n        with torch.no_grad():\n            logits = model(imgs.to(device))\n            \n        # We can still compute the loss (but not the gradient).\n        loss = criterion(logits, labels.to(device))\n\n        # Compute the accuracy for current batch.\n        acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n\n        # Record the loss and accuracy.\n        valid_loss.append(loss.item())\n        valid_accs.append(acc)\n        \n    # The average loss and accuracy for entire validation set is the average of the recorded values.\n    valid_loss = sum(valid_loss) / len(valid_loss)\n    valid_acc = sum(valid_accs) / len(valid_accs)\n\n    # Print the information.\n    print(f\"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}, lr = {optimizer.param_groups[0]['lr']:.5f}\")\n\n    # if the model improves, save a checkpoint at this epoch\n    if valid_acc > best_acc:\n        best_acc = valid_acc\n        torch.save(model.state_dict(), model_path)\n        print('saving model with acc {:.3f}'.format(best_acc))\n\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-28T09:22:57.452044Z","iopub.execute_input":"2021-06-28T09:22:57.4524Z","iopub.status.idle":"2021-06-28T09:23:14.653982Z","shell.execute_reply.started":"2021-06-28T09:22:57.452363Z","shell.execute_reply":"2021-06-28T09:23:14.652047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"预测阶段","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom PIL import Image\nimport os\nimport matplotlib.pyplot as plt\nimport torchvision.models as models\n# This is for the progress bar.\nfrom tqdm import tqdm\n\n\ndef get_device():\n    return 'cuda' if torch.cuda.is_available() else 'cpu'\n\ndevice = get_device()\ntorch.backends.cudnn.benchmark = True\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:49:58.044484Z","iopub.execute_input":"2021-06-29T06:49:58.044825Z","iopub.status.idle":"2021-06-29T06:49:59.292192Z","shell.execute_reply.started":"2021-06-29T06:49:58.044751Z","shell.execute_reply":"2021-06-29T06:49:59.291307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 看看label文件长啥样\nlabels_dataframe = pd.read_csv('../input/classify-leaves/train.csv')\n\n\n# 把label文件排个序\nleaves_labels = sorted(list(set(labels_dataframe['label'])))\nn_classes = len(leaves_labels)\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nnum_to_class = {v : k for k, v in class_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:50:15.912093Z","iopub.execute_input":"2021-06-29T06:50:15.91244Z","iopub.status.idle":"2021-06-29T06:50:15.955535Z","shell.execute_reply.started":"2021-06-29T06:50:15.912401Z","shell.execute_reply":"2021-06-29T06:50:15.954713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass LeavesData(Dataset):\n    def __init__(self, csv_path, file_path, mode='train', valid_ratio=0.2, resize_height=256, resize_width=256):\n        \"\"\"\n        Args:\n            csv_path (string): csv 文件路径\n            img_path (string): 图像文件所在路径\n            mode (string): 训练模式还是测试模式\n            valid_ratio (float): 验证集比例\n        \"\"\"\n        \n        # 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#\n        self.resize_height = resize_height\n        self.resize_width = resize_width\n\n        self.file_path = file_path\n        self.mode = mode\n\n        # 读取 csv 文件\n        # 利用pandas读取csv文件\n        self.data_info = pd.read_csv(csv_path, header=None)  #header=None是去掉表头部分\n        # 计算 length\n        self.data_len = len(self.data_info.index) - 1\n        self.train_len = int(self.data_len * (1 - valid_ratio))\n\n        \n\n        if mode == 'train':\n            # 第一列包含图像文件的名称\n            self.train_image = np.asarray(self.data_info.iloc[1:self.train_len, 0])  #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始到train_len\n            # 第二列是图像的 label\n            self.train_label = np.asarray(self.data_info.iloc[1:self.train_len, 1])\n            self.image_arr = self.train_image \n            self.label_arr = self.train_label\n        elif mode == 'valid':\n            self.valid_image = np.asarray(self.data_info.iloc[self.train_len:, 0])  \n            self.valid_label = np.asarray(self.data_info.iloc[self.train_len:, 1])\n            self.image_arr = self.valid_image\n            self.label_arr = self.valid_label\n        elif mode == 'test':\n            self.test_image = np.asarray(self.data_info.iloc[1:, 0])\n            self.image_arr = self.test_image\n            \n        self.real_len = len(self.image_arr)\n\n        print('Finished reading the {} set of Leaves Dataset ({} samples found)'\n              .format(mode, self.real_len))\n\n    def __getitem__(self, index):\n        # 从 image_arr中得到索引对应的文件名\n        single_image_name = self.image_arr[index]\n\n        # 读取图像文件\n        img_as_img = Image.open(self.file_path + single_image_name)\n\n        #如果需要将RGB三通道的图片转换成灰度图片可参考下面两行\n#         if img_as_img.mode != 'L':\n#             img_as_img = img_as_img.convert('L')\n\n        #设置好需要转换的变量，还可以包括一系列的nomarlize等等操作\n        if self.mode == 'train':\n            transform = transforms.Compose([\n                transforms.Resize((224, 224)),\n                transforms.RandomHorizontalFlip(p=0.5),              \n                #transforms.RandomResizedCrop(size=224, scale=(0.8, 1.0)),\n                #transforms.RandomRotation(degrees=15),\n                #transforms.RandomHorizontalFlip(),\n                #transforms.CenterCrop(size=224),\n                transforms.ToTensor(),\n                #transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n            ])\n        else:\n            # valid和test不做数据增强\n            transform = transforms.Compose([\n                transforms.Resize((224, 224)),\n                transforms.CenterCrop(size=224),\n                transforms.ToTensor(),\n                #transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n            ])\n        \n        img_as_img = transform(img_as_img)\n        \n        if self.mode == 'test':\n            return img_as_img\n        else:\n            # 得到图像的 string label\n            label = self.label_arr[index]\n            # number label\n            number_label = class_to_num[label]\n\n            return img_as_img, number_label  #返回每一个index对应的图片数据和对应的label\n\n    def __len__(self):\n        return self.real_len\n","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:50:33.691991Z","iopub.execute_input":"2021-06-29T06:50:33.692313Z","iopub.status.idle":"2021-06-29T06:50:33.707614Z","shell.execute_reply.started":"2021-06-29T06:50:33.692282Z","shell.execute_reply":"2021-06-29T06:50:33.706678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_path = '../input/classify-leaves/train.csv'\ntest_path = '../input/classify-leaves/test.csv'\n# csv文件中已经images的路径了，因此这里只到上一级目录\nimg_path = '../input/classify-leaves/'","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:51:21.567956Z","iopub.execute_input":"2021-06-29T06:51:21.568381Z","iopub.status.idle":"2021-06-29T06:51:21.576379Z","shell.execute_reply.started":"2021-06-29T06:51:21.568344Z","shell.execute_reply":"2021-06-29T06:51:21.575302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = LeavesData(train_path, img_path, mode='train')\nval_dataset = LeavesData(train_path, img_path, mode='valid')\ntest_dataset = LeavesData(test_path, img_path, mode='test')","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:51:26.614069Z","iopub.execute_input":"2021-06-29T06:51:26.614421Z","iopub.status.idle":"2021-06-29T06:51:26.671177Z","shell.execute_reply.started":"2021-06-29T06:51:26.614368Z","shell.execute_reply":"2021-06-29T06:51:26.670157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# 定义data loader\ntrain_loader = torch.utils.data.DataLoader(\n        dataset=train_dataset,\n        batch_size=224, \n        shuffle=True,\n        num_workers=8,\n        pin_memory=True\n    )\n\nval_loader = torch.utils.data.DataLoader(\n        dataset=val_dataset,\n        batch_size=224, \n        shuffle=False,\n        num_workers=8,\n        pin_memory=True\n    )\ntest_loader = torch.utils.data.DataLoader(\n        dataset=test_dataset,\n        batch_size=64, \n        shuffle=False,\n        num_workers=8,\n        pin_memory=True\n    )\n \n# 是否要冻住模型的前面一些层\ndef set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        model = model\n        for param in model.parameters():\n            param.requires_grad = False\n# resnet152模型\ndef res_model(num_classes, feature_extract = False, use_pretrained=True):\n\n    model_ft = models.resnext101_32x8d(pretrained=use_pretrained)\n    set_parameter_requires_grad(model_ft, feature_extract)\n    num_ftrs = model_ft.fc.in_features\n    model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))\n\n    return model_ft\n","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:51:46.88735Z","iopub.execute_input":"2021-06-29T06:51:46.887738Z","iopub.status.idle":"2021-06-29T06:51:46.895855Z","shell.execute_reply.started":"2021-06-29T06:51:46.887706Z","shell.execute_reply":"2021-06-29T06:51:46.895025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel_path = './input/resnext101-32x8d/pre_res_model2.ckpt'\n\n\nsaveFileName = './2submission.csv'\n\n## predict\nmodel = res_model(176)\n\n# print(model)\n# input()\n# create model and load weights from checkpoint\nmodel = nn.DataParallel(model).cuda()\nmodel.load_state_dict(torch.load(model_path))\n\n\n# Make sure the model is in eval mode.\n# Some modules like Dropout or BatchNorm affect if the model is in training mode.\nmodel.eval()\n\n# Initialize a list to store the predictions.\npredictions = []\n# Iterate the testing set by batches.\nfor batch in tqdm(test_loader):\n    \n    imgs = batch\n    with torch.no_grad():\n        logits = model(imgs.to(device))\n    \n    # Take the class with greatest logit as prediction and record it.\n    predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())\n\npreds = []\nfor i in predictions:\n    preds.append(num_to_class[i])\n\n\n\ntest_data = pd.read_csv(test_path)\ntest_data['label'] = pd.Series(preds)\nsubmission = pd.concat([test_data['image'], test_data['label']], axis=1)\nsubmission.to_csv(saveFileName, index=False)\nprint(\"Done!!!!!!!!!!!!!!!!!!!!!!!!!!!\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T06:51:58.021168Z","iopub.execute_input":"2021-06-29T06:51:58.021506Z","iopub.status.idle":"2021-06-29T06:52:19.629208Z","shell.execute_reply.started":"2021-06-29T06:51:58.021473Z","shell.execute_reply":"2021-06-29T06:52:19.627241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}