{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"经过两三年的kaggle经历，我把比赛tricks沉淀成了一个库[TorchUtils](https://github.com/seefun/TorchUtils)，仍在更新中，这个代码也是用了这个库来快速构建一个strong [baseline](https://github.com/seefun/TorchUtils/blob/master/examples/kaggle_leaves_classification.ipynb).\n\n由于kaggle notebook训练时间限制，无法在时限内完成训练，故需要本地安装依赖并训练测试.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-26T08:16:27.812601Z","iopub.execute_input":"2021-06-26T08:16:27.813042Z","iopub.status.idle":"2021-06-26T08:16:27.818568Z","shell.execute_reply.started":"2021-06-26T08:16:27.812954Z","shell.execute_reply":"2021-06-26T08:16:27.817475Z"}}},{"cell_type":"markdown","source":"## Prepare Libs\n首先,导入依赖库","metadata":{}},{"cell_type":"code","source":"# First, you should install TorchUtils (clone and pip install: https://github.com/seefun/TorchUtils)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T08:23:38.436752Z","iopub.execute_input":"2021-06-26T08:23:38.437128Z","iopub.status.idle":"2021-06-26T08:23:38.441936Z","shell.execute_reply.started":"2021-06-26T08:23:38.437099Z","shell.execute_reply":"2021-06-26T08:23:38.440443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport time\nimport math\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.optim import Adam, AdamW\nfrom torch.nn.parameter import Parameter\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport torch.nn.functional as F\nimport seaborn as sns\nimport random\nimport sys\nimport gc\nimport shutil\nfrom tqdm.autonotebook import tqdm\nimport albumentations\nfrom albumentations import pytorch as AT\n\nimport scipy.special\nsigmoid = lambda x: scipy.special.expit(x)\nfrom scipy.special import softmax\n\nimport torch_utils as tu \n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"为了保证实验的可复现性，我们通常需要固定seed，并使用torch的deterministic功能","metadata":{}},{"cell_type":"code","source":"SEED = 42\nbase_dir = '../input/'\ntu.tools.seed_everything(SEED, deterministic=False)\ntu.tools.set_gpus('0') # choice gpu id if you have more than one GPU","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"自动备份实验，保证试验记录不丢失","metadata":{}},{"cell_type":"code","source":"EXP = 1\nwhile os.path.exists('../exp/exp%d'%EXP):\n    EXP+=1\nos.makedirs('../exp/exp%d'%EXP)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Param\n定义可调参数","metadata":{}},{"cell_type":"code","source":"CLASSES = 176\nFOLD = 5\nBATCH_SIZE = 64\nACCUMULATE = 1\nLR = 3e-4\nEPOCH = 36\nDECAY_SCALE = 20.0\nMIXUP = 0 # 0 to 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset\n\n定义数据集以及增广，使用了RandAugment","metadata":{}},{"cell_type":"code","source":"train_transform = albumentations.Compose([\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.0625, rotate_limit=45, border_mode=1, p=0.5),\n    tu.randAugment(),\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n    \ntest_transform = albumentations.Compose([\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n\n\nclass LeavesDataset(Dataset):\n    \n    def __init__(self, df, label_encoder, data_path='../input', transform = train_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n        self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path, label = self.df.image[idx], self.df.label[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img, label","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# K Fold Split Train-Val dataset\nsfolder = StratifiedKFold(n_splits=FOLD,random_state=SEED,shuffle=True)\ntr_folds = []\nval_folds = []\nfor train_idx, val_idx in sfolder.split(train_df.image, train_df.label):\n    tr_folds.append(train_idx)\n    val_folds.append(val_idx)\n    print(len(train_idx), len(val_idx))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Loop\n\n混合精度训练，节约显存来获得更大batch size，并节约训练时间，损失较小，甚至能因为大batch而提升性能","metadata":{}},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR\nscaler = torch.cuda.amp.GradScaler() # for AMP training ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(epoch, verbose=False):\n    model_conv.train()         \n    avg_loss = 0.\n    optimizer.zero_grad()\n    if verbose:\n        bar = tqdm(total=len(train_loader))\n    mixup_fn = tu.Mixup(prob=MIXUP, switch_prob=0.0, onehot=True, label_smoothing=0.05, num_classes=CLASSES)\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.float().cuda(), labels.cuda()\n        if MIXUP:\n            imgs_train, labels_train = mixup_fn(imgs_train, labels_train)\n        with torch.cuda.amp.autocast():\n            output_train, _ = model_conv(imgs_train)\n            loss = criterion(output_train, labels_train)\n        scaler.scale(loss).backward()\n        if ((idx+1)%ACCUMULATE==0): # Gradient Accumulate\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            scheduler.step()\n        avg_loss += loss.item() / len(train_loader) \n        if verbose:\n            bar.update(1)\n    if verbose:\n        bar.close()\n    return avg_loss\n\ndef test_model():    \n    avg_val_loss = 0.\n    model_conv.eval()\n    y_true_val = np.zeros(len(valset))\n    y_pred_val = np.zeros((len(valset), CLASSES))\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_vaild, labels_vaild = imgs.float().cuda(), labels.cuda()\n            output_test, _ = model_conv(imgs_vaild)\n            avg_val_loss += (criterion_test(output_test, labels_vaild).item() / len(val_loader)) \n            a = labels_vaild.detach().cpu().numpy().astype(np.int)\n            b = softmax(output_test.detach().cpu().numpy(), axis=1)\n\n            y_true_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = a\n            y_pred_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = b\n            \n    metric_val = sum(np.argmax(y_pred_val, axis=1) == y_true_val) / len(y_true_val)\n    return avg_val_loss, metric_val","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fold):\n    best_avg_loss = 100.0\n    best_acc = 0.0\n\n    avg_val_loss, avg_val_acc = test_model()\n    print('pretrain val loss %.4f precision %.4f'%(avg_val_loss, avg_val_acc))       \n\n    ### training\n    for epoch in range(EPOCH):   \n        print('lr:', optimizer.param_groups[0]['lr']) \n        np.random.seed(SEED+EPOCH*999)\n        start_time = time.time()\n        avg_loss = train_model(epoch)\n        avg_val_loss, avg_val_acc = test_model()\n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t train_loss={:.4f} \\t val_loss={:.4f} \\t val_precision={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, EPOCH, avg_loss, avg_val_loss, avg_val_acc, elapsed_time))\n\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n\n        if avg_val_acc > best_acc:\n            best_acc = avg_val_acc\n            torch.save(model_conv.module.state_dict(), '../exp/exp' + str(EXP) + '/model-best' + str(fold) + '.pth')\n            print('model saved!')\n\n        print('=================================')   \n\n    print('best loss:', best_avg_loss)\n    print('best precision:', best_acc)\n    return best_avg_loss, best_acc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. 使用imagenet-21k上预训练的efficientnetv2模型：https://arxiv.org/abs/2104.00298\n2. mutli-dropout \n3. pooling时concat maxpooling与avgpooling\n4. mixup / Label Smoothing作为正则\n5. 余弦学习率下降\n6. 使用Ranger([RAdam](arxiv.org/abs/1908.03265)+[Lookahead](https://arxiv.org/abs/1907.08610)+[GC](https://arxiv.org/abs/2004.01461))/ RangerLars(Ranger+Lars)优化器","metadata":{}},{"cell_type":"code","source":"log = open('../exp/exp' + str(EXP) +'/log.txt', 'w')\nlog.write('SEED%d\\n'%SEED)\ncv_losses = []\ncv_metrics = []\n\nfor fold in range(FOLD):\n    print('\\n ********** Fold %d **********\\n'%fold)\n    ###################### Dataset #######################\n    labels = train_df.label.unique()\n    label_encoder = {}\n    for idx, name in enumerate(labels):\n        label_encoder.update({name:idx})\n    \n    trainset = LeavesDataset(train_df.iloc[tr_folds[fold]].reset_index(), label_encoder, base_dir, train_transform)\n    train_loader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, num_workers=16, shuffle=True, drop_last=True, worker_init_fn=tu.tools.worker_init_fn)\n    \n    valset = LeavesDataset(train_df.iloc[val_folds[fold]].reset_index(), label_encoder, base_dir, test_transform)\n    val_loader = torch.utils.data.DataLoader(valset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)\n    \n    ####################### Model ########################\n    model_conv = tu.ImageModel(name='tf_efficientnetv2_l_in21ft1k', pretrained=True, feature=2048, classes=CLASSES)\n    model_conv.cuda()\n    model_conv = torch.nn.DataParallel(model_conv)\n\n    ###################### Optim ########################\n    optimizer = tu.RangerLars(model_conv.parameters(), lr=LR, weight_decay=2e-4)\n\n    if MIXUP:\n        criterion = tu.SoftTargetCrossEntropy()\n    else:\n        criterion = tu.LabelSmoothingCrossEntropy()\n        \n    criterion_test = nn.CrossEntropyLoss()\n\n    T = len(train_loader)//ACCUMULATE * EPOCH # cycle\n    scheduler = CosineAnnealingLR(optimizer, T_max=T, eta_min=LR/DECAY_SCALE)\n    \n    val_loss, val_acc = train(fold)\n    \n    cv_losses.append(val_loss)\n    cv_metrics.append(val_acc)\n    torch.cuda.empty_cache()\n\ncv_loss = sum(cv_losses) / FOLD\ncv_acc = sum(cv_metrics) / FOLD\nprint('CV loss:%.6f  CV precision:%.6f'%(cv_loss, cv_acc))\nlog.write('CV loss:%.6f  CV precision:%.6f\\n\\n'%(cv_loss, cv_acc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"实验结束后自动备份代码和结果","metadata":{}},{"cell_type":"code","source":"log.close()\ntu.tools.backup_folder('.', '../exp/exp%d/src'%EXP)  # backup code","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"markdown","source":"使用TTA来获得更好的结果，详见改比赛[讨论区](https://www.kaggle.com/c/classify-leaves/discussion/245198)\n\nTTA一般是必定有用的，代价是更长的inference时间，在此次比赛中，各种正则化防止过拟合的方法之所以不work，大概率是因为:\n1. 比赛数据集存在大量leak，训练测试集大量图片重复或接近，过拟合反而能获得更好结果；\n2. 数据集存在很多噪声样本，过拟合这些错误标注反而能获得更高的LB；即LB本身并不可靠，更高的LB不代表真正更高的泛化性能；","metadata":{}},{"cell_type":"code","source":"class LeavesDataset_inference(Dataset):\n    \n    def __init__(self, df, data_path='../input', transform = test_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path = self.df.image[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img\n\ntest_df = pd.read_csv(os.path.join(base_dir, 'test.csv'))\ntestset = LeavesDataset_inference(test_df, base_dir, tta_transform)\ntest_loader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TTA = 16\npred_list = []\n\nfor tta in range(TTA):\n    for fold in range(FOLD):\n        y_pred_val = np.zeros((len(testset), CLASSES))\n        model_conv = tu.ImageModel(name='tf_efficientnetv2_l_in21ft1k', pretrained=False, feature=2048, classes=CLASSES)\n        model_conv.cuda()\n        model_conv.load_state_dict(torch.load('../exp/exp' + str(EXP) + '/model-best' + str(fold) + '.pth'))\n        with torch.no_grad():\n            for idx, imgs in enumerate(test_loader):\n                imgs_test = imgs.float().cuda()\n                output_test, _ = model_conv(imgs_test)\n                b = softmax(output_test.detach().cpu().numpy(), axis=1)\n                y_pred_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = b\n        pred_list.append(y_pred_val)\n        print('TTA %d: Fold %d'%(tta,fold))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get submission\npred = sum(pred_list)/len(pred_list)\nnp.save('../exp/exp' + str(EXP) + '/pred.npy', pred)\n\nlabel_decoder = {}\nfor k, v in label_encoder.items():\n    label_decoder[v] = k\n    \npred_label = np.argmax(pred, axis=1)\ntest_df['label'] = pred_label\ntest_df['label'] = test_df['label'].apply(lambda x: label_decoder[x])\n\ntest_df.to_csv('exp%d.csv'%EXP, index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"最后，单模型可以获得的结果是，0.98727 prvate LB | 0.98795 public LB","metadata":{}}]}