{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"12th--Public Score Score:0.98659，Private Score:0.98704, reference to seefun","metadata":{}},{"cell_type":"markdown","source":"首先感谢seefun，最后的结果都是基于seefun的代码和torch—utils上微调得到的。","metadata":{}},{"cell_type":"markdown","source":"我的base-line是基于resnest50上得到的结果，没有做mixup和torch-utils的randAugment（也就是说数据增强不够强），没有用K折交叉验证。最后的公榜分数0.97000，私榜分数0.97204。看到seefun的discussion后采用其框架，score直接涨了0.005，太香了，遂用其框架进行微调。","metadata":{}},{"cell_type":"markdown","source":"**声明：以下代码均为seefun的代码，我只是在上面进行了微调。**\nhttps://github.com/seefun/TorchUtils","metadata":{}},{"cell_type":"markdown","source":"## Prepare Lib","metadata":{}},{"cell_type":"code","source":"# First, you should install TorchUtils (see README.md)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport time\nimport math\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.optim import Adam, AdamW\nfrom torch.nn.parameter import Parameter\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport torch.nn.functional as F\nimport seaborn as sns\nimport random\nimport sys\nimport gc\nimport shutil\nfrom tqdm.autonotebook import tqdm\nimport albumentations\nfrom albumentations import pytorch as AT\n\nimport scipy.special\nsigmoid = lambda x: scipy.special.expit(x)\nfrom scipy.special import softmax\n\nimport torch_utils as tu \n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nbase_dir = '../input/'\ntu.tools.seed_everything(SEED, deterministic=False)\ntu.tools.set_gpus('0') # gpu ids","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXP = 1\nwhile os.path.exists('../exp/exp%d'%EXP):\n    EXP+=1\nos.makedirs('../exp/exp%d'%EXP)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Param","metadata":{}},{"cell_type":"markdown","source":"我觉得36个epoch还没有完全收敛，于是将epoch增加到72. 五折交叉验证后的score相对于36个epoch增加了0.001.\n\n加入mixup，设为0.1（太大会使模型性能下降）","metadata":{}},{"cell_type":"code","source":"CLASSES = 176\nFOLD = 5\nBATCH_SIZE = 64\nACCUMULATE = 1\nLR = 3e-4\nEPOCH = 72\nDECAY_SCALE = 20.0\nMIXUP = 0.1 # 0 to 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"markdown","source":"在seefun代码基础上，在randAugment中加入cut_out，效果很玄学，有一点点的提升","metadata":{}},{"cell_type":"code","source":"train_transform = albumentations.Compose([\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.0625, rotate_limit=45, border_mode=1, p=0.5),\n    tu.randAugment(N=2,M=6,p=1,cut_out=True),\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n    \ntest_transform = albumentations.Compose([\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n\n\nclass LeavesDataset(Dataset):\n    \n    def __init__(self, df, label_encoder, data_path='../input', transform = train_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n        self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path, label = self.df.image[idx], self.df.label[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img, label","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sfolder = StratifiedKFold(n_splits=FOLD,random_state=SEED,shuffle=True)\ntr_folds = []\nval_folds = []\nfor train_idx, val_idx in sfolder.split(train_df.image, train_df.label):\n    tr_folds.append(train_idx)\n    val_folds.append(val_idx)\n    print(len(train_idx), len(val_idx))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR\nscaler = torch.cuda.amp.GradScaler() # for AMP training","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(epoch, verbose=False):\n    model_conv.train()         \n    avg_loss = 0.\n    optimizer.zero_grad()\n    if verbose:\n        bar = tqdm(total=len(train_loader))\n    mixup_fn = tu.Mixup(prob=MIXUP, switch_prob=0.0, onehot=True, label_smoothing=0.05, num_classes=CLASSES)\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.float().cuda(), labels.cuda()\n        if MIXUP:\n            imgs_train, labels_train = mixup_fn(imgs_train, labels_train)\n        with torch.cuda.amp.autocast():\n            output_train, _ = model_conv(imgs_train)\n            loss = criterion(output_train, labels_train)\n        scaler.scale(loss).backward()\n        if ((idx+1)%ACCUMULATE==0): # Gradient Accumulate\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            scheduler.step()\n        avg_loss += loss.item() / len(train_loader) \n        if verbose:\n            bar.update(1)\n    if verbose:\n        bar.close()\n    return avg_loss\n\ndef test_model():    \n    avg_val_loss = 0.\n    model_conv.eval()\n    y_true_val = np.zeros(len(valset))\n    y_pred_val = np.zeros((len(valset), CLASSES))\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_vaild, labels_vaild = imgs.float().cuda(), labels.cuda()\n            output_test, _ = model_conv(imgs_vaild)\n            avg_val_loss += (criterion_test(output_test, labels_vaild).item() / len(val_loader)) \n            a = labels_vaild.detach().cpu().numpy().astype(np.int)\n            b = softmax(output_test.detach().cpu().numpy(), axis=1)\n\n            y_true_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = a\n            y_pred_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = b\n            \n    metric_val = sum(np.argmax(y_pred_val, axis=1) == y_true_val) / len(y_true_val)\n    return avg_val_loss, metric_val","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fold):\n    best_avg_loss = 100.0\n    best_acc = 0.0\n\n    avg_val_loss, avg_val_acc = test_model()\n    print('pretrain val loss %.4f precision %.4f'%(avg_val_loss, avg_val_acc))       \n\n    ### training\n    for epoch in range(EPOCH):   \n        print('lr:', optimizer.param_groups[0]['lr']) \n        np.random.seed(SEED+EPOCH*999)\n        start_time = time.time()\n        avg_loss = train_model(epoch)\n        avg_val_loss, avg_val_acc = test_model()\n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t train_loss={:.4f} \\t val_loss={:.4f} \\t val_precision={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, EPOCH, avg_loss, avg_val_loss, avg_val_acc, elapsed_time))\n\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n\n        if avg_val_acc > best_acc:\n            best_acc = avg_val_acc\n            torch.save(model_conv.module.state_dict(), '../exp/exp' + str(EXP) + '/model-best' + str(fold) + '.pth')\n            print('model saved!')\n\n        print('=================================')   \n\n    print('best loss:', best_avg_loss)\n    print('best precision:', best_acc)\n    return best_avg_loss, best_acc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"resnest的狂热粉丝，使用resnest50d的预训练模型：https://hangzhang.org/files/resnest.pdf\n\nmixup + Label Smoothing\n\n使用Ranger([RAdam](arxiv.org/abs/1908.03265)+[Lookahead](https://arxiv.org/abs/1907.08610)+[GC](https://arxiv.org/abs/2004.01461))/ RangerLars(Ranger+Lars)优化器(这个优化器感觉收敛的更快）","metadata":{}},{"cell_type":"code","source":"log = open('../exp/exp' + str(EXP) +'/log.txt', 'w')\nlog.write('SEED%d\\n'%SEED)\ncv_losses = []\ncv_metrics = []\n\nfor fold in range(FOLD):\n    print('\\n ********** Fold %d **********\\n'%fold)\n    ###################### Dataset #######################\n    if os.path.exist('../exp/exp' + str(EXP) + '/model-best{}.pth'.format(fold)):\n        print('fold{} has been trained!'.format(fold))\n        continue\n    labels = train_df.label.unique()\n    label_encoder = {}\n    for idx, name in enumerate(labels):\n        label_encoder.update({name:idx})\n    \n    trainset = LeavesDataset(train_df.iloc[tr_folds[fold]].reset_index(), label_encoder, base_dir, train_transform)\n    train_loader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, num_workers=16, shuffle=True, drop_last=True, worker_init_fn=tu.tools.worker_init_fn)\n    \n    valset = LeavesDataset(train_df.iloc[val_folds[fold]].reset_index(), label_encoder, base_dir, test_transform)\n    val_loader = torch.utils.data.DataLoader(valset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)\n    \n    ####################### Model ########################\n    model_conv = tu.ImageModel(name='resnest50d', pretrained=True, feature=2048, classes=CLASSES)\n    model_conv.cuda()\n    model_conv = torch.nn.DataParallel(model_conv)\n\n    ###################### Optim ########################\n    optimizer = tu.RangerLars(model_conv.parameters(), lr=LR, weight_decay=2e-4)\n\n    if MIXUP:\n        criterion = tu.SoftTargetCrossEntropy()\n    else:\n        criterion = tu.LabelSmoothingCrossEntropy()\n        \n    criterion_test = nn.CrossEntropyLoss()\n\n    T = len(train_loader)//ACCUMULATE * EPOCH # cycle\n    scheduler = CosineAnnealingLR(optimizer, T_max=T, eta_min=LR/DECAY_SCALE)\n    \n    val_loss, val_acc = train(fold)\n    \n    cv_losses.append(val_loss)\n    cv_metrics.append(val_acc)\n    torch.cuda.empty_cache()\n    log.write('fold{} best precision{} \\n'.format(fold,val_acc))\n\ncv_loss = sum(cv_losses) / FOLD\ncv_acc = sum(cv_metrics) / FOLD\nprint('CV loss:%.6f  CV precision:%.6f'%(cv_loss, cv_acc))\nlog.write('CV loss:%.6f  CV precision:%.6f\\n\\n'%(cv_loss, cv_acc))","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log.close()\ntu.tools.backup_folder('.', '../exp/exp%d/src'%EXP)  # backup code","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"markdown","source":"我只是简单的使用了5折交叉验证然后结果取出现最多的label","metadata":{}},{"cell_type":"code","source":"class LeavesDataset_test(Dataset):\n    \n    def __init__(self, df, label_encoder, data_path='../input', transform = test_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n        #self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path = self.df.image[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img\n\ndef encode_label(df):\n    labels = df.label.unique()\n    class_to_num = {}\n    for idx, name in enumerate(labels):\n        class_to_num.update({name: idx})\n    num_to_class = {v: k for k, v in class_to_num.items()}\n    return class_to_num,num_to_class","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ntrain_path = '../input/classify-leaves/train.csv'\ntest_path = '../input/classify-leaves/train.csv'\nimg_path = '../input/classify-leaves/images'\nsaveFileName = '../exp/exp{}/model_all'.format(EXP)\nmodel_path = '../exp/exp{}/model-best'.format(EXP)\n\nFOLD = 5\n\n\ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)\nclass_to_num, num_to_class = encode_label(train_df)\n\n\ntest_dataset = LeavesDataset_test(test_df)\n\n\ntest_loader = torch.utils.data.DataLoader(\n        dataset=test_dataset,\n        batch_size=64,\n        shuffle=False,\n        num_workers=8\n    )\nprint(len(test_loader))\n## predict\n#model = res_model(176)\nmodel = tu.ImageModel(name='resnest50d', pretrained=True, feature=2048, classes=176)\n# create model and load weights from checkpoint\nmodel = model.to(device)\npredictions_folder = []\nfor fold in range(FOLD):\n\n    model.load_state_dict(torch.load(model_path+'{}.pth'.format(fold)))\n    model.eval()\n\n    predictions = []\n    for batch in tqdm(test_loader): \n\n        imgs = batch\n        with torch.no_grad():\n            logits,_ = model(imgs.to(device))\n            #print(logits)\n        predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())\n    print(predictions)\n    predictions_folder.append(predictions)\nprediction = []\npredictions_folder = np.array(predictions_folder)\nfor i in range(len(predictions_folder[0])):\n    max_label = np.argmax(np.bincount(predictions_folder[:,i]))\n    prediction.append(max_label)\nprint(prediction)\npreds = []\nfor j in prediction:\n    preds.append(num_to_class[j])\n\ntest_data = pd.read_csv(test_path)\ntest_data['label'] = pd.Series(preds)\nsubmission = pd.concat([test_data['image'], test_data['label']], axis=1)\nsubmission.to_csv(saveFileName, index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"取五折交叉验证的结果是0.98704（私榜），0.98659（公榜）。五折交叉验证会使模型性能提高0.003","metadata":{}}]}