{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:37.162883Z","iopub.execute_input":"2021-06-27T05:29:37.163251Z","iopub.status.idle":"2021-06-27T05:29:44.451284Z","shell.execute_reply.started":"2021-06-27T05:29:37.163171Z","shell.execute_reply":"2021-06-27T05:29:44.45047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn, optim\nfrom torchvision import transforms\nimport timm\n\nfrom sklearn.model_selection import KFold\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport time","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:44.453111Z","iopub.execute_input":"2021-06-27T05:29:44.45341Z","iopub.status.idle":"2021-06-27T05:29:47.582Z","shell.execute_reply.started":"2021-06-27T05:29:44.453381Z","shell.execute_reply":"2021-06-27T05:29:47.581202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据分割","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv(\"../input/classify-leaves/train.csv\")\n\nleaves_labels = sorted(list(set(train_csv['label'])))\nn_classes = len(leaves_labels)\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nnum_to_class = {v : k for k, v in class_to_num.items()}\n\ndef kfold(data, k=5):\n    \"\"\" K折交叉验证 \"\"\"\n    \n    KF = KFold(n_splits=k, shuffle=False)\n    for train_idxs, test_idxs in KF.split(data):\n        train_data = data.loc[train_idxs].reset_index(drop=True)\n        valid_data = data.loc[test_idxs].reset_index(drop=True)\n        train_iter = torch.utils.data.DataLoader(\n            ReadData(train_data, train_transform), batch_size=64,\n            shuffle=True, num_workers=3, pin_memory=True\n        )\n\n        valid_iter = torch.utils.data.DataLoader(\n            ReadData(valid_data, valid_transform), batch_size=64,\n            shuffle=True, num_workers=3, pin_memory=True\n        )\n        \n        yield train_iter, valid_iter","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.584281Z","iopub.execute_input":"2021-06-27T05:29:47.584546Z","iopub.status.idle":"2021-06-27T05:29:47.631087Z","shell.execute_reply.started":"2021-06-27T05:29:47.584521Z","shell.execute_reply":"2021-06-27T05:29:47.630138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取数据","metadata":{}},{"cell_type":"code","source":"class ReadData(torch.utils.data.Dataset):\n    def __init__(self, csv_data, transform=None):\n        super(ReadData, self).__init__()\n        self.data = csv_data\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img = Image.open(\"../input/classify-leaves/\" + self.data.loc[idx, \"image\"])\n        label = class_to_num[self.data.loc[idx, \"label\"]]\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img, label\n        \n    def __len__(self):\n        return len(self.data)\n    \ntrain_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(p=.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nvalid_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.632533Z","iopub.execute_input":"2021-06-27T05:29:47.632912Z","iopub.status.idle":"2021-06-27T05:29:47.642657Z","shell.execute_reply.started":"2021-06-27T05:29:47.632868Z","shell.execute_reply":"2021-06-27T05:29:47.641646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for k, (train_iter, valid_iter) in enumerate(kfold(train_csv, 5)):\n#     for x, y in train_iter:\n#         print(x.size())","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.646332Z","iopub.execute_input":"2021-06-27T05:29:47.646726Z","iopub.status.idle":"2021-06-27T05:29:47.653382Z","shell.execute_reply.started":"2021-06-27T05:29:47.646668Z","shell.execute_reply":"2021-06-27T05:29:47.652642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据增强","metadata":{}},{"cell_type":"markdown","source":"## 图像叠加","metadata":{}},{"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n    \"\"\" Mixup 数据增强 -> 随机叠加两张图像 \"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    \n    return mixed_x, y_a, y_b, lam","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.656649Z","iopub.execute_input":"2021-06-27T05:29:47.65692Z","iopub.status.idle":"2021-06-27T05:29:47.665338Z","shell.execute_reply.started":"2021-06-27T05:29:47.656896Z","shell.execute_reply":"2021-06-27T05:29:47.664406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 图像翻转","metadata":{}},{"cell_type":"code","source":"def flip_data(x, y, alpha=1.0, use_cuda=True):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n        \n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n    new = transforms.RandomRotation(degrees=(90, 180))\n    new_x = new(x)\n    y_a, y_b = y, y[index]\n    return new_x, y_a, y_b, lam\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.666596Z","iopub.execute_input":"2021-06-27T05:29:47.667078Z","iopub.status.idle":"2021-06-27T05:29:47.675459Z","shell.execute_reply.started":"2021-06-27T05:29:47.667039Z","shell.execute_reply":"2021-06-27T05:29:47.674496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 随机裁剪","metadata":{}},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    \n    W, H = size[2], size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix_data(x, y, alpha=1.0, use_cuda=True):\n    \"\"\" Cutmix 数据增强 -> 随机对主图像进行裁剪, 加上噪点图像\n    W: 添加裁剪图像宽\n    H: 添加裁剪图像高\n    \"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n    \n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    y_a, y_b = y, y[index]\n    \n    return x, y_a, y_b, lam","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.677032Z","iopub.execute_input":"2021-06-27T05:29:47.677409Z","iopub.status.idle":"2021-06-27T05:29:47.689445Z","shell.execute_reply.started":"2021-06-27T05:29:47.67737Z","shell.execute_reply":"2021-06-27T05:29:47.688587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_models(k=5):\n    models = {}\n    for mk in range(k):\n        model = timm.create_model(\"resnest50d_4s2x40d\", False, drop_rate=.5)\n        model.fc = nn.Sequential(\n            nn.Linear(model.fc.in_features, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(.3),\n            nn.Linear(512, len(num_to_class))\n        )\n        # for param in model.layer4.parameters():\n        #     if isinstance(param, nn.Conv2d):\n        #         nn.init.xavier_normal_(param.weight)\n        # for param in model.fc.parameters():\n        #     if isinstance(param, nn.Linear):\n        #         nn.init.kaiming_normal_(param.weight)\n        model.load_state_dict(torch.load(f\"../input/resnest50dnew/Resnest50d_new.pth\"))\n        for i, param in enumerate(model.children()):\n            if i == 6:\n                break\n            param.requires_grad = False\n\n        model.cuda()\n\n        opt = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4)\n        scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, 10, T_mult=2)\n        models[f\"model_{mk}\"] = {\n            \"model\": model,\n            \"opt\": opt,\n            \"scheduler\": scheduler,\n            \"last_acc\": .97\n        }\n    \n    return models\n\nmodels = get_models()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:47.690739Z","iopub.execute_input":"2021-06-27T05:29:47.691234Z","iopub.status.idle":"2021-06-27T05:29:58.404592Z","shell.execute_reply.started":"2021-06-27T05:29:47.691193Z","shell.execute_reply":"2021-06-27T05:29:58.403716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup_criterion(pred, y_a, y_b, lam):\n    c = nn.CrossEntropyLoss()\n    return lam * c(pred, y_a) + (1 - lam) * c(pred, y_b)\ncriterion = mixup_criterion","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.40603Z","iopub.execute_input":"2021-06-27T05:29:58.406394Z","iopub.status.idle":"2021-06-27T05:29:58.412651Z","shell.execute_reply.started":"2021-06-27T05:29:58.406346Z","shell.execute_reply":"2021-06-27T05:29:58.411668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练","metadata":{}},{"cell_type":"code","source":"def train_model():\n    for epoch in range(20):\n        flod_train_acc = []\n        flod_valid_acc = []\n        for k, (train_iter, valid_iter) in enumerate(kfold(train_csv, 5)):\n            model = models[f\"model_{k}\"][\"model\"]\n            opt = models[f\"model_{k}\"][\"opt\"]\n            scheduler = models[f\"model_{k}\"][\"scheduler\"]\n            s = time.time()\n            model.train()\n            train_loss = []\n            train_acc = 0\n            length = 0\n            for x, y in train_iter:\n                x, y = x.cuda(), y.cuda()\n                random_num = np.random.random()\n                if random_num <= 1/4:\n                    x, y_a, y_b, lam = mixup_data(x, y, use_cuda=True)\n                elif random_num <= 1/2:\n                    x, y_a, y_b, lam = cutmix_data(x, y, use_cuda=True)\n                elif random_num <= 3/4:\n                    x, y_a, y_b, lam = flip_data(x, y, use_cuda=True)\n                else \n                    x, y_a, y_b, lam = mixup_data(x, y, alpha=0, use_cuda=True)\n                x, y_a, y_b = map(torch.autograd.Variable, (x, y_a, y_b))\n                output = model(x)\n                loss = criterion(output, y_a, y_b, lam)\n                train_loss.append(loss.item())\n                predict = output.argmax(dim=1)\n                length += x.shape[0]\n                train_acc += lam * (predict == y_a).cpu().sum().item() + \\\n                            (1 - lam) * (predict == y_b).cpu().sum().item()\n                opt.zero_grad()\n                loss.backward()\n                opt.step()\n                scheduler.step()\n\n            model.eval()\n            valid_acc = []\n            with torch.no_grad():\n                for x, y in valid_iter:\n                    x, y = x.cuda(), y.cuda()\n                    pre_x = model(x)\n                    valid_acc.append((pre_x.argmax(1) == y).float().mean().item())\n\n            k_train_ = train_acc / length\n            k_valid_ = sum(valid_acc) / len(valid_acc)\n            if k_valid_ > models[f\"model_{k}\"][\"last_acc\"]:\n                torch.save(model.state_dict(), f\"/kaggle/working/Resnest50d_{k}.pth\")\n                models[f\"model_{k}\"][\"last_acc\"] = k_valid_\n\n            response = f\"Epoch {epoch + 1}-Fold{k + 1} —— \" + \\\n                    f\"Train Loss: {sum(train_loss) / len(train_loss) :.3f}, \" + \\\n                    f\"Train Accuracy: {k_train_ * 100 :.2f}%, \" + \\\n                    f\"Valid Accuracy: {k_valid_ * 100 :.2f}%, \" + \\\n                    f\"Learning Rate: {opt.param_groups[0]['lr'] :.6f}, \" + \\\n                    f\"Time Out: {time.time() - s :.1f}s\"\n            print(response)\n            flod_train_acc.append(k_train_)\n            flod_valid_acc.append(k_valid_)\n\n        t_accuracy = np.mean(flod_train_acc)\n        v_accuracy = np.mean(flod_valid_acc)\n        print(f\"Epoch {epoch + 1} —— \" + \\\n            f\"Train Accuracy: {t_accuracy * 100 :.2f}%, \" + \\\n            f\"Valid Accuracy: {v_accuracy * 100 :.2f}%\\n\")\n\ntrain_model()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.414727Z","iopub.execute_input":"2021-06-27T05:29:58.415122Z","iopub.status.idle":"2021-06-27T05:29:58.434691Z","shell.execute_reply.started":"2021-06-27T05:29:58.415082Z","shell.execute_reply":"2021-06-27T05:29:58.433403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TestModel","metadata":{}},{"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/classify-leaves/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.435905Z","iopub.status.idle":"2021-06-27T05:29:58.43674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ReadData(torch.utils.data.Dataset):\n    def __init__(self, csv_data, transform=None):\n        super(ReadData, self).__init__()\n        self.data = csv_data\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img = Image.open(\"../input/classify-leaves/\" + self.data.loc[idx, \"image\"])\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img\n        \n    def __len__(self):\n        return len(self.data)\n\ntest_iter = torch.utils.data.DataLoader(\n    ReadData(test_csv, valid_transform), batch_size=64,\n    num_workers=2, pin_memory=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.438117Z","iopub.status.idle":"2021-06-27T05:29:58.438914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = None\nwith torch.no_grad():\n    for x in test_iter:\n        x = x.cuda()\n        p = torch.zeros((x.size()[0], len(class_to_num)))\n        for k in range(5):\n            model = models[f\"model_{k}\"][\"model\"]\n            p += model(x).detach().cpu()\n        if predict is None:\n            predict = p.argmax(1)\n        else:\n            predict = torch.cat([predict, p.argmax(1)])","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.440159Z","iopub.status.idle":"2021-06-27T05:29:58.441078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/classify-leaves/sample_submission.csv\")\ndf.label = predict.cpu().numpy()\ndf.label = df.label.apply(lambda x: num_to_class[x])\ndf.to_csv(\"/kaggle/working/result.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T05:29:58.442327Z","iopub.status.idle":"2021-06-27T05:29:58.443109Z"},"trusted":true},"execution_count":null,"outputs":[]}]}