{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"第一次参加kaggle的比赛，选用的是EfficientNet-b1的模型，非常感谢seefun提供的一些trick，在未使用数据增强、5折交叉验证、mixup、labelsmoothing、CosineAnnealingLR得到的结果是0.96340, 本代码是在seefun的基础上进行修改\n\n比赛结果为：\n+ private learnboard：0.98590\n+ public leaderboard：0.98363","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-01T14:18:37.265074Z","iopub.execute_input":"2021-07-01T14:18:37.265513Z","iopub.status.idle":"2021-07-01T14:18:37.27508Z","shell.execute_reply.started":"2021-07-01T14:18:37.265467Z","shell.execute_reply":"2021-07-01T14:18:37.273918Z"}}},{"cell_type":"markdown","source":"# step1: 安装相关依赖\n\n+ 首先需要根据 https://github.com/seefun/TorchUtils 安装torch_utils工具包\n+ 使用的是albumentations数据增强库\n+ 使用StratifiedKFold 5折交叉验证\n+ 安装efficientnet_pytorch\n+ 加载其他依赖项等","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/torch-utils/TorchUtils')\n!pip install -r /kaggle/input/torch-utils/TorchUtils/requirements.txt\n!pip install /kaggle/input/torch-utils/TorchUtils","metadata":{"execution":{"iopub.status.busy":"2021-07-01T16:09:42.341669Z","iopub.execute_input":"2021-07-01T16:09:42.34243Z","iopub.status.idle":"2021-07-01T16:09:58.731078Z","shell.execute_reply.started":"2021-07-01T16:09:42.34237Z","shell.execute_reply":"2021-07-01T16:09:58.729556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# step2: 加载数据","metadata":{}},{"cell_type":"code","source":"import torch_utils as tu\n\nSEED = 20\ntu.tools.seed_everything(SEED, deterministic=False) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = 176\nFOLD = 5\nBATCH_SIZE = 128\nACCUMULATE = 1\nLR = 0.01\nEPOCH = 50\nDECAY_SCALE = 20.0\nMIXUP = 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntrain_df = pd.read_csv('/kaggle/input/classify-leaves/train.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nsfolder = StratifiedKFold(n_splits=FOLD,random_state=SEED,shuffle=True)\ntr_folds = []\nval_folds = []\nfor train_idx, val_idx in sfolder.split(train_df.image, train_df.label):\n    tr_folds.append(train_idx)\n    val_folds.append(val_idx)\n    print(len(train_idx), len(val_idx))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations\nfrom albumentations import pytorch as AT\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\n\ntrain_transform = albumentations.Compose([\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.0625, rotate_limit=45, border_mode=1, p=0.5),\n    tu.randAugment(),\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n])\n    \ntest_transform = albumentations.Compose([\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n])\n\nclass LeavesDataset(Dataset):\n    \n    def __init__(self, df, label_encoder, data_path='./data', transform = train_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n        self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path, label = self.df.image[idx], self.df.label[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img, label","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# step3: 开始训练模型","metadata":{}},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR\nscaler = torch.cuda.amp.GradScaler() # for AMP training","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(epoch, verbose=False):\n    model_conv.train()         \n    avg_loss = 0.\n    optimizer.zero_grad()\n    if verbose:\n        bar = tqdm(total=len(train_loader))\n    mixup_fn = tu.Mixup(prob=MIXUP, switch_prob=0.0, onehot=True, label_smoothing=0.05, num_classes=CLASSES)\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.float().cuda(), labels.cuda()\n        if MIXUP:\n            imgs_train, labels_train = mixup_fn(imgs_train, labels_train)\n        with torch.cuda.amp.autocast():\n            output_train = model_conv(imgs_train)\n            loss = criterion(output_train, labels_train)\n        scaler.scale(loss).backward()\n        if ((idx+1)%ACCUMULATE==0): # Gradient Accumulate\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            scheduler.step()\n        avg_loss += loss.item() / len(train_loader) \n        if verbose:\n            bar.update(1)\n    if verbose:\n        bar.close()\n    return avg_loss\n\ndef test_model():    \n    avg_val_loss = 0.\n    model_conv.eval()\n    y_true_val = np.zeros(len(valset))\n    y_pred_val = np.zeros((len(valset), CLASSES))\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_vaild, labels_vaild = imgs.float().cuda(), labels.cuda()\n            output_test = model_conv(imgs_vaild)\n            avg_val_loss += (criterion_test(output_test, labels_vaild).item() / len(val_loader)) \n            a = labels_vaild.detach().cpu().numpy().astype(np.int)\n            b = softmax(output_test.detach().cpu().numpy(), axis=1)\n\n            y_true_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = a\n            y_pred_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = b\n            \n    metric_val = sum(np.argmax(y_pred_val, axis=1) == y_true_val) / len(y_true_val)\n    return avg_val_loss, metric_val","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fold):\n    best_avg_loss = 100.0\n    best_acc = 0.0\n\n    avg_val_loss, avg_val_acc = test_model()\n    print('pretrain val loss %.4f precision %.4f'%(avg_val_loss, avg_val_acc))       \n\n    ### training\n    for epoch in range(EPOCH):   \n        print('lr:', optimizer.param_groups[0]['lr']) \n        np.random.seed(SEED+EPOCH*999)\n        start_time = time.time()\n        avg_loss = train_model(epoch)\n        avg_val_loss, avg_val_acc = test_model()\n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t train_loss={:.4f} \\t val_loss={:.4f} \\t val_precision={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, EPOCH, avg_loss, avg_val_loss, avg_val_acc, elapsed_time))\n\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n\n        if avg_val_acc > best_acc:\n            best_acc = avg_val_acc\n            torch.save(model_conv.state_dict(), './model-best' + str(fold) + '.pth')\n            print('model saved!')\n\n        print('=================================') \n\n    print('best loss:', best_avg_loss)\n    print('best precision:', best_acc)\n    return best_avg_loss, best_acc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nimport numpy as np\nimport cv2\nfrom scipy.special import softmax\nimport time\n\ncv_losses = []\ncv_metrics = []\n\nfor fold in range(FOLD):\n    print('\\n ********** Fold %d **********\\n'%fold)\n\n    labels = train_df.label.unique()\n    label_encoder = {}\n    for idx, name in enumerate(labels):\n        label_encoder.update({name:idx})\n    \n    trainset = LeavesDataset(train_df.iloc[tr_folds[fold]].reset_index(), label_encoder, base_dir, train_transform)\n    train_loader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, num_workers=4, shuffle=True, drop_last=True, worker_init_fn=tu.tools.worker_init_fn)\n    \n    valset = LeavesDataset(train_df.iloc[val_folds[fold]].reset_index(), label_encoder, base_dir, test_transform)\n    val_loader = torch.utils.data.DataLoader(valset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n    model_conv = EfficientNet.from_pretrained('efficientnet-b1', num_classes=CLASSES)\n    model_conv.cuda()\n\n    optimizer = tu.RangerLars(model_conv.parameters(), lr=LR, weight_decay=1e-4)\n\n    if MIXUP:\n        criterion = tu.SoftTargetCrossEntropy()\n    else:\n        criterion = tu.LabelSmoothingCrossEntropy()\n        \n    criterion_test = nn.CrossEntropyLoss()\n\n    T = len(train_loader)//ACCUMULATE * EPOCH # cycle\n    scheduler = CosineAnnealingLR(optimizer, T_max=T, eta_min=LR/DECAY_SCALE)\n    \n    val_loss, val_acc = train(fold)\n    \n    cv_losses.append(val_loss)\n    cv_metrics.append(val_acc)\n    torch.cuda.empty_cache()\n\ncv_loss = sum(cv_losses) / FOLD\ncv_acc = sum(cv_metrics) / FOLD\nprint('CV loss:%.6f  CV precision:%.6f'%(cv_loss, cv_acc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# step4：模型验证","metadata":{}},{"cell_type":"code","source":"class LeavesDatasets(Dataset):\n    def __init__(self, df, data_path='./data', transform = train_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path = self.df.image[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv( '/kaggle/input/classify-leaves/test.csv')\ntest_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testset = LeavesDatasets(test_df, base_dir, test_transform)\ntest_loader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm.notebook as tqdm\nfrom efficientnet_pytorch import EfficientNet\nimport numpy as np\nimport cv2\nfrom scipy.special import softmax\nimport time\n\ntst_preds_all = []\n\nfor fold in range(FOLD):\n    model = EfficientNet.from_pretrained('efficientnet-b1', num_classes=CLASSES)\n    model.cuda()\n    \n    model.load_state_dict(torch.load('./model-best' + str(fold) + '.pth'))\n    \n    with torch.no_grad():\n        model.eval()\n        image_preds_all = []\n        \n        for idx, imgs in enumerate(test_loader):\n            imgs = imgs.cuda()\n            image_preds = model(imgs)\n            image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]      \n        \n        image_preds_all = np.concatenate(image_preds_all, axis=0)\n\n    tst_preds_all.append(image_preds_all)\n\n    del model\n    torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_tst = np.mean(tst_preds_all, axis=0)\ntest_df['label'] = np.argmax(avg_tst, axis=1)\ntest_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_df.label.unique()\nlabel_encoder = {}\nfor idx, name in enumerate(labels):\n    label_encoder.update({idx:name})\n    \ntest_df['label'] = test_df['label'].apply(lambda x: label_encoder[x])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"./result.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}