{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:41.449145Z","iopub.execute_input":"2021-06-20T11:54:41.449556Z","iopub.status.idle":"2021-06-20T11:54:49.627505Z","shell.execute_reply.started":"2021-06-20T11:54:41.449470Z","shell.execute_reply":"2021-06-20T11:54:49.626349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn, optim\nfrom torchvision import transforms\nimport timm\n\nfrom sklearn.model_selection import KFold\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport time","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:49.631365Z","iopub.execute_input":"2021-06-20T11:54:49.631642Z","iopub.status.idle":"2021-06-20T11:54:52.750840Z","shell.execute_reply.started":"2021-06-20T11:54:49.631613Z","shell.execute_reply":"2021-06-20T11:54:52.749913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据分割","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv(\"../input/classify-leaves/train.csv\")\n\nleaves_labels = sorted(list(set(train_csv['label'])))\nn_classes = len(leaves_labels)\nclass_to_num = dict(zip(leaves_labels, range(n_classes)))\nnum_to_class = {v : k for k, v in class_to_num.items()}\n\n# def train_test_split(data, train_size=.75):\n#     \"\"\" 分割数据集 \n#     return: train_data_df, valid_data_df\n#     \"\"\"\n#     train_len = int(len(data) * train_size)\n#     indexs = np.arange(len(data))\n#     np.random.shuffle(indexs)\n\n#     return data.loc[indexs[:train_len]].reset_index(drop=True),\\\n#            data.loc[indexs[train_len:]].reset_index(drop=True)\n\n# train_csv, valid_csv = train_test_split(train_csv)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:52.752274Z","iopub.execute_input":"2021-06-20T11:54:52.752614Z","iopub.status.idle":"2021-06-20T11:54:52.794115Z","shell.execute_reply.started":"2021-06-20T11:54:52.752579Z","shell.execute_reply":"2021-06-20T11:54:52.793242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取数据","metadata":{}},{"cell_type":"code","source":"class ReadData(torch.utils.data.Dataset):\n    def __init__(self, csv_data, transform=None):\n        super(ReadData, self).__init__()\n        self.data = csv_data\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img = Image.open(\"../input/classify-leaves/\" + self.data.loc[idx, \"image\"])\n        label = class_to_num[self.data.loc[idx, \"label\"]]\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img, label\n        \n    def __len__(self):\n        return len(self.data)\n    \ntrain_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(p=.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nvalid_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\ndef kfold(data, k=5):\n    \"\"\" K折交叉验证 \"\"\"\n    \n    KF = KFold(n_splits=k, shuffle=False)\n    for train_idxs, test_idxs in KF.split(data):\n        train_data = data.loc[train_idxs].reset_index(drop=True)\n        valid_data = data.loc[test_idxs].reset_index(drop=True)\n        train_iter = torch.utils.data.DataLoader(\n            ReadData(train_data, train_transform), batch_size=64,\n            shuffle=True, num_workers=3, pin_memory=True\n        )\n\n        valid_iter = torch.utils.data.DataLoader(\n            ReadData(valid_data, valid_transform), batch_size=64,\n            shuffle=True, num_workers=3, pin_memory=True\n        )\n        \n        yield train_iter, valid_iter","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:52.795586Z","iopub.execute_input":"2021-06-20T11:54:52.795930Z","iopub.status.idle":"2021-06-20T11:54:52.808044Z","shell.execute_reply.started":"2021-06-20T11:54:52.795892Z","shell.execute_reply":"2021-06-20T11:54:52.807262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据增强","metadata":{}},{"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n    \"\"\" Mixup 数据增强 -> 随机叠加两张图像 \"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:52.810726Z","iopub.execute_input":"2021-06-20T11:54:52.811373Z","iopub.status.idle":"2021-06-20T11:54:52.820334Z","shell.execute_reply.started":"2021-06-20T11:54:52.811337Z","shell.execute_reply":"2021-06-20T11:54:52.819566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    \"\"\" 随机裁剪 \"\"\"\n    W, H = size[2], size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix_data(x, y, alpha=1.0, use_cuda=True):\n    \"\"\" Cutmix 数据增强 -> 随机对主图像进行裁剪, 加上噪点图像\n    W: 添加裁剪图像宽\n    H: 添加裁剪图像高\n    \"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n    \n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    y_a, y_b = y, y[index]\n    \n    return x, y_a, y_b, lam","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:52.821988Z","iopub.execute_input":"2021-06-20T11:54:52.823397Z","iopub.status.idle":"2021-06-20T11:54:52.836039Z","shell.execute_reply.started":"2021-06-20T11:54:52.823339Z","shell.execute_reply":"2021-06-20T11:54:52.835061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_models(k=5):\n    models = {}\n    for mk in range(k):\n        model = timm.create_model(\"resnest50d_4s2x40d\", False, drop_rate=.5)\n        model.fc = nn.Sequential(\n            nn.Linear(model.fc.in_features, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(.3),\n            nn.Linear(512, len(num_to_class))\n        )\n        # for param in model.layer4.parameters():\n        #     if isinstance(param, nn.Conv2d):\n        #         nn.init.xavier_normal_(param.weight)\n        # for param in model.fc.parameters():\n        #     if isinstance(param, nn.Linear):\n        #         nn.init.kaiming_normal_(param.weight)\n        model.load_state_dict(torch.load(f\"../input/resnest50/Resnest50d.pth\"))\n        for i, param in enumerate(model.children()):\n            if i == 6:\n                break\n            param.requires_grad = False\n\n        model.cuda()\n\n        opt = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-5)\n        scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, 10, T_mult=2)\n        models[f\"model_{mk}\"] = {\n            \"model\": model,\n            \"opt\": opt,\n            \"scheduler\": scheduler,\n            \"last_acc\": .97\n        }\n    \n    return models\n\nmodels = get_models()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:54:52.837687Z","iopub.execute_input":"2021-06-20T11:54:52.838075Z","iopub.status.idle":"2021-06-20T11:55:03.293378Z","shell.execute_reply.started":"2021-06-20T11:54:52.838038Z","shell.execute_reply":"2021-06-20T11:55:03.292496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup_criterion(pred, y_a, y_b, lam):\n    c = nn.CrossEntropyLoss()\n    return lam * c(pred, y_a) + (1 - lam) * c(pred, y_b)\ncriterion = mixup_criterion","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:55:03.294964Z","iopub.execute_input":"2021-06-20T11:55:03.295289Z","iopub.status.idle":"2021-06-20T11:55:03.299446Z","shell.execute_reply.started":"2021-06-20T11:55:03.295254Z","shell.execute_reply":"2021-06-20T11:55:03.298627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练","metadata":{}},{"cell_type":"code","source":"def train_model():\n    for epoch in range(10):\n        flod_train_acc = []\n        flod_valid_acc = []\n        for k, (train_iter, valid_iter) in enumerate(kfold(train_csv, 5)):\n            model = models[f\"model_{k}\"][\"model\"]\n            opt = models[f\"model_{k}\"][\"opt\"]\n            scheduler = models[f\"model_{k}\"][\"scheduler\"]\n            s = time.time()\n            model.train()\n            train_loss = []\n            train_acc = 0\n            length = 0\n            for x, y in train_iter:\n                x, y = x.cuda(), y.cuda()\n                random_num = np.random.random()\n                if random_num <= 1/3:\n                    x, y_a, y_b, lam = mixup_data(x, y, use_cuda=True)\n                elif random_num <= 2/3:\n                    x, y_a, y_b, lam = cutmix_data(x, y, use_cuda=True)\n                else:\n                    x, y_a, y_b, lam = mixup_data(x, y, alpha=0, use_cuda=True)\n                x, y_a, y_b = map(torch.autograd.Variable, (x, y_a, y_b))\n                output = model(x)\n                loss = criterion(output, y_a, y_b, lam)\n                train_loss.append(loss.item())\n                predict = output.argmax(dim=1)\n                length += x.shape[0]\n                train_acc += lam * (predict == y_a).cpu().sum().item() + \\\n                            (1 - lam) * (predict == y_b).cpu().sum().item()\n                opt.zero_grad()\n                loss.backward()\n                opt.step()\n                scheduler.step()\n\n            model.eval()\n            valid_acc = []\n            with torch.no_grad():\n                for x, y in valid_iter:\n                    x, y = x.cuda(), y.cuda()\n                    pre_x = model(x)\n                    valid_acc.append((pre_x.argmax(1) == y).float().mean().item())\n\n            k_train_ = train_acc / length\n            k_valid_ = sum(valid_acc) / len(valid_acc)\n            if k_valid_ > models[f\"model_{k}\"][\"last_acc\"]:\n                torch.save(model.state_dict(), f\"/kaggle/working/Resnest50d_{k}.pth\")\n                models[f\"model_{k}\"][\"last_acc\"] = k_valid_\n\n            response = f\"Epoch {epoch + 1}-Fold{k + 1} —— \" + \\\n                    f\"Train Loss: {sum(train_loss) / len(train_loss) :.3f}, \" + \\\n                    f\"Train Accuracy: {k_train_ * 100 :.2f}%, \" + \\\n                    f\"Valid Accuracy: {k_valid_ * 100 :.2f}%, \" + \\\n                    f\"Learning Rate: {opt.param_groups[0]['lr'] :.6f}, \" + \\\n                    f\"Time Out: {time.time() - s :.1f}s\"\n            print(response)\n            flod_train_acc.append(k_train_)\n            flod_valid_acc.append(k_valid_)\n\n        t_accuracy = np.mean(flod_train_acc)\n        v_accuracy = np.mean(flod_valid_acc)\n        print(f\"Epoch {epoch + 1} —— \" + \\\n            f\"Train Accuracy: {t_accuracy * 100 :.2f}%, \" + \\\n            f\"Valid Accuracy: {v_accuracy * 100 :.2f}%\\n\")\n\ntrain_model()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:51:22.617301Z","iopub.execute_input":"2021-06-20T11:51:22.617652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TestModel","metadata":{}},{"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/classify-leaves/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:55:14.530994Z","iopub.execute_input":"2021-06-20T11:55:14.531344Z","iopub.status.idle":"2021-06-20T11:55:14.548351Z","shell.execute_reply.started":"2021-06-20T11:55:14.531310Z","shell.execute_reply":"2021-06-20T11:55:14.547390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ReadData(torch.utils.data.Dataset):\n    def __init__(self, csv_data, transform=None):\n        super(ReadData, self).__init__()\n        self.data = csv_data\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img = Image.open(\"../input/classify-leaves/\" + self.data.loc[idx, \"image\"])\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img\n        \n    def __len__(self):\n        return len(self.data)\n\ntest_iter = torch.utils.data.DataLoader(\n    ReadData(test_csv, valid_transform), batch_size=64,\n    num_workers=2, pin_memory=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:55:14.828006Z","iopub.execute_input":"2021-06-20T11:55:14.828531Z","iopub.status.idle":"2021-06-20T11:55:14.835699Z","shell.execute_reply.started":"2021-06-20T11:55:14.828492Z","shell.execute_reply":"2021-06-20T11:55:14.834695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = None\nwith torch.no_grad():\n    for x in test_iter:\n        x = x.cuda()\n        p = torch.zeros((x.size()[0], len(class_to_num)))\n        for k in range(5):\n            model = models[f\"model_{k}\"][\"model\"]\n            p += model(x).detach().cpu()\n        if predict is None:\n            predict = p.argmax(1)\n        else:\n            predict = torch.cat([predict, p.argmax(1)])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:56:15.765958Z","iopub.execute_input":"2021-06-20T11:56:15.766311Z","iopub.status.idle":"2021-06-20T11:58:18.810992Z","shell.execute_reply.started":"2021-06-20T11:56:15.766281Z","shell.execute_reply":"2021-06-20T11:58:18.809969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/classify-leaves/sample_submission.csv\")\ndf.label = predict.cpu().numpy()\ndf.label = df.label.apply(lambda x: num_to_class[x])\ndf.to_csv(\"/kaggle/working/result.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:58:18.812873Z","iopub.execute_input":"2021-06-20T11:58:18.813218Z","iopub.status.idle":"2021-06-20T11:58:19.113630Z","shell.execute_reply.started":"2021-06-20T11:58:18.813175Z","shell.execute_reply":"2021-06-20T11:58:19.112867Z"},"trusted":true},"execution_count":null,"outputs":[]}]}