import os import time import math import random import numpy as np import pandas as pd from pathlib import Path import glob import matplotlib.pyplot as plt from PIL import Image, ImageEnhance, ImageOps from tqdm import tqdm, tqdm_notebook import torch from torch import nn, cuda from torch.autograd import Variable import torch.nn.functional as F import torchvision as vision import torchvision.models as models from torch.utils.data import Dataset, DataLoader from torch.optim import Adam, SGD, Optimizer from torch.optim.lr_scheduler import _LRScheduler, CosineAnnealingLR, ReduceLROnPlateau from sklearn.metrics import f1_score os.environ['CUDA_LAUNCH_BLOCKING'] = "1" def get_base_model(base_model, num_classes): if(base_model == 'ResNet-50'): model_ft = models.resnet50(pretrained=True) model_ft.fc = nn.Linear(2048, num_classes) return model_ft, (224, 224) elif(base_model == 'ResNext50'): model_ft = models.resnext50_32x4d(pretrained=True) num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) elif(base_model == 'ResNext101'): model_ft = models.resnext101_32x8d(pretrained=True) num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) elif(base_model == 'ResNet-101'): model_ft = models.resnet101(pretrained=True) num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) elif(base_model == 'VGG19-bn'): model_ft = models.vgg19_bn(pretrained=True) num_ftrs = model_ft.classifier[6].in_features model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) elif(base_model == 'ResNext101_32x8d_wsl'): model_ft = torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x8d_wsl') num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) elif(base_model == 'ResNext101_32x16d_wsl'): model_ft = torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl') num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) return model_ft, (224, 224) def load_model(base_model, num_classes, weight_path): if(base_model == 'ResNet-50'): model_ft = models.resnet50() model_ft.fc = nn.Linear(2048, num_classes) model_ft.load_state_dict(torch.load(weight_path)) return model_ft, (224, 224) elif(base_model == 'ResNext50'): model_ft = models.resnext50_32x4d() num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) model_ft.load_state_dict(torch.load(weight_path)) return model_ft, (224, 224) elif(base_model == 'ResNext101'): model_ft = models.resnext101_32x8d() num_ftrs = model_ft.fc.in_features model_ft.fc = nn.Linear(num_ftrs, num_classes) model_ft.load_state_dict(torch.load(weight_path)) return model_ft, (224, 224) class AdamW(Optimizer): """Implements AdamW algorithm. It has been proposed in `Fixing Weight Decay Regularization in Adam`_. Arguments: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-3) betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) .. Fixing Weight Decay Regularization in Adam: https://arxiv.org/abs/1711.05101 """ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) super(AdamW, self).__init__(params, defaults) def step(self, closure=None): """Performs a single optimization step. Arguments: closure (callable, optional): A closure that reevaluates the model and returns the loss. """ loss = None if closure is not None: loss = closure() for group in self.param_groups: for p in group['params']: if p.grad is None: continue grad = p.grad.data if grad.is_sparse: raise RuntimeError('AdamW does not support sparse gradients, please consider SparseAdam instead') state = self.state[p] # State initialization if len(state) == 0: state['step'] = 0 # Exponential moving average of gradient values state['exp_avg'] = torch.zeros_like(p.data) # Exponential moving average of squared gradient values state['exp_avg_sq'] = torch.zeros_like(p.data) exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] beta1, beta2 = group['betas'] state['step'] += 1 # according to the paper, this penalty should come after the bias correction # if group['weight_decay'] != 0: # grad = grad.add(group['weight_decay'], p.data) # Decay the first and second moment running average coefficient exp_avg.mul_(beta1).add_(1 - beta1, grad) exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) denom = exp_avg_sq.sqrt().add_(group['eps']) bias_correction1 = 1 - beta1 ** state['step'] bias_correction2 = 1 - beta2 ** state['step'] step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 p.data.addcdiv_(-step_size, exp_avg, denom) if group['weight_decay'] != 0: p.data.add_(-group['weight_decay'], p.data) return loss class CosineAnnealingWarmRestarts(_LRScheduler): # https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html r"""Set the learning rate of each parameter group using a cosine annealing schedule, where :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}` is the number of epochs since the last restart and :math:`T_{i}` is the number of epochs between two warm restarts in SGDR: .. math:: \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 + \cos(\frac{T_{cur}}{T_{i}}\pi)) When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`. When :math:`T_{cur}=0`(after restart), set :math:`\eta_t=\eta_{max}`. It has been proposed in `SGDR: Stochastic Gradient Descent with Warm Restarts`_. Args: optimizer (Optimizer): Wrapped optimizer. T_0 (int): Number of iterations for the first restart. T_mult (int, optional): A factor increases :math:`T_{i}` after a restart. Default: 1. eta_min (float, optional): Minimum learning rate. Default: 0. last_epoch (int, optional): The index of last epoch. Default: -1. .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: https://arxiv.org/abs/1608.03983 """ def __init__(self, optimizer, T_0, T_mult=1, eta_min=0, last_epoch=-1, restart_decay=1): if T_0 <= 0 or not isinstance(T_0, int): raise ValueError("Expected positive integer T_0, but got {}".format(T_0)) if T_mult < 1 or not isinstance(T_mult, int): raise ValueError("Expected integer T_mul >= 1, but got {}".format(T_mul)) self.T_0 = T_0 self.T_i = T_0 self.T_mult = T_mult self.eta_min = eta_min self.T_num = 0 #추가 self.restart_decay = restart_decay #추가 super(CosineAnnealingWarmRestarts, self).__init__(optimizer, last_epoch) self.T_cur = last_epoch def get_lr(self): return [self.eta_min + (base_lr*self.restart_decay**self.T_num - self.eta_min) * (1 + math.cos(math.pi * self.T_cur / self.T_i)) / 2 for base_lr in self.base_lrs] def step(self, epoch=None): """Step could be called after every update, i.e. if one epoch has 10 iterations (number_of_train_examples / batch_size), we should call SGDR.step(0.1), SGDR.step(0.2), etc. This function can be called in an interleaved way. Example: >>> scheduler = SGDR(optimizer, T_0, T_mult) >>> for epoch in range(20): >>> scheduler.step() >>> scheduler.step(26) >>> scheduler.step() # scheduler.step(27), instead of scheduler(20) """ if epoch is None: epoch = self.last_epoch + 1 self.T_cur = self.T_cur + 1 if self.T_cur >= self.T_i: self.T_cur = self.T_cur - self.T_i self.T_i = self.T_i * self.T_mult self.T_num += 1 # 추가 else: if epoch >= self.T_0: if self.T_mult == 1: self.T_cur = epoch % self.T_0 else: n = int(math.log((epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult)) self.T_cur = epoch - self.T_0 * (self.T_mult ** n - 1) / (self.T_mult - 1) self.T_i = self.T_0 * self.T_mult ** (n) else: self.T_i = self.T_0 self.T_cur = epoch self.last_epoch = math.floor(epoch) for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): param_group['lr'] = lr ''' 유명한 mixup 논문. 타대회에서도 많이 씁니다. https://arxiv.org/abs/1710.09412 ''' def mixup_criterion(criterion, pred, y_a, y_b, lam): return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) def mixup_data(x, y, alpha=1.0, use_cuda=True): '''Returns mixed inputs, pairs of targets, and lambda''' if alpha > 0: lam = np.random.beta(alpha, alpha) else: lam = 1 batch_size = x.size()[0] if use_cuda: index = torch.randperm(batch_size).cuda() else: index = torch.randperm(batch_size) mixed_x = lam * x + (1 - lam) * x[index, :] y_a, y_b = y, y[index] return mixed_x, y_a, y_b, lam class SVHNPolicy(object): """ Randomly choose one of the best 25 Sub-policies on SVHN. Example: >>> policy = SVHNPolicy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> SVHNPolicy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): self.policies = [ SubPolicy(0.9, "shearX", 4, 0.2, "invert", 3, fillcolor), SubPolicy(0.9, "shearY", 8, 0.7, "invert", 5, fillcolor), SubPolicy(0.6, "equalize", 5, 0.6, "solarize", 6, fillcolor), SubPolicy(0.9, "invert", 3, 0.6, "equalize", 3, fillcolor), SubPolicy(0.6, "equalize", 1, 0.9, "rotate", 3, fillcolor), SubPolicy(0.9, "shearX", 4, 0.8, "autocontrast", 3, fillcolor), SubPolicy(0.9, "shearY", 8, 0.4, "invert", 5, fillcolor), SubPolicy(0.9, "shearY", 5, 0.2, "solarize", 6, fillcolor), SubPolicy(0.9, "invert", 6, 0.8, "autocontrast", 1, fillcolor), SubPolicy(0.6, "equalize", 3, 0.9, "rotate", 3, fillcolor), SubPolicy(0.9, "shearX", 4, 0.3, "solarize", 3, fillcolor), SubPolicy(0.8, "shearY", 8, 0.7, "invert", 4, fillcolor), SubPolicy(0.9, "equalize", 5, 0.6, "translateY", 6, fillcolor), SubPolicy(0.9, "invert", 4, 0.6, "equalize", 7, fillcolor), SubPolicy(0.3, "contrast", 3, 0.8, "rotate", 4, fillcolor), SubPolicy(0.8, "invert", 5, 0.0, "translateY", 2, fillcolor), SubPolicy(0.7, "shearY", 6, 0.4, "solarize", 8, fillcolor), SubPolicy(0.6, "invert", 4, 0.8, "rotate", 4, fillcolor), SubPolicy(0.3, "shearY", 7, 0.9, "translateX", 3, fillcolor), SubPolicy(0.1, "shearX", 6, 0.6, "invert", 5, fillcolor), SubPolicy(0.7, "solarize", 2, 0.6, "translateY", 7, fillcolor), SubPolicy(0.8, "shearY", 4, 0.8, "invert", 8, fillcolor), SubPolicy(0.7, "shearX", 9, 0.8, "translateY", 3, fillcolor), SubPolicy(0.8, "shearY", 5, 0.7, "autocontrast", 3, fillcolor), SubPolicy(0.7, "shearX", 2, 0.1, "invert", 5, fillcolor) ] def __call__(self, img): policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment SVHN Policy" class ImageNetPolicy(object): """ Randomly choose one of the best 24 Sub-policies on ImageNet. Example: >>> policy = ImageNetPolicy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> ImageNetPolicy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): self.policies = [ SubPolicy(0.4, "posterize", 8, 0.6, "rotate", 9, fillcolor), SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor), SubPolicy(0.6, "posterize", 7, 0.6, "posterize", 6, fillcolor), SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), SubPolicy(0.4, "equalize", 4, 0.8, "rotate", 8, fillcolor), SubPolicy(0.6, "solarize", 3, 0.6, "equalize", 7, fillcolor), SubPolicy(0.8, "posterize", 5, 1.0, "equalize", 2, fillcolor), SubPolicy(0.2, "rotate", 3, 0.6, "solarize", 8, fillcolor), SubPolicy(0.6, "equalize", 8, 0.4, "posterize", 6, fillcolor), SubPolicy(0.8, "rotate", 8, 0.4, "color", 0, fillcolor), SubPolicy(0.4, "rotate", 9, 0.6, "equalize", 2, fillcolor), SubPolicy(0.0, "equalize", 7, 0.8, "equalize", 8, fillcolor), SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), SubPolicy(0.8, "rotate", 8, 1.0, "color", 2, fillcolor), SubPolicy(0.8, "color", 8, 0.8, "solarize", 7, fillcolor), SubPolicy(0.4, "sharpness", 7, 0.6, "invert", 8, fillcolor), SubPolicy(0.6, "shearX", 5, 1.0, "equalize", 9, fillcolor), SubPolicy(0.4, "color", 0, 0.6, "equalize", 3, fillcolor), SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor) ] def __call__(self, img): policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment ImageNet Policy" class CIFAR10Policy(object): """ Randomly choose one of the best 25 Sub-policies on CIFAR10. Example: >>> policy = CIFAR10Policy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> CIFAR10Policy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): self.policies = [ SubPolicy(0.1, "invert", 7, 0.2, "contrast", 6, fillcolor), SubPolicy(0.7, "rotate", 2, 0.3, "translateX", 9, fillcolor), SubPolicy(0.8, "sharpness", 1, 0.9, "sharpness", 3, fillcolor), SubPolicy(0.5, "shearY", 8, 0.7, "translateY", 9, fillcolor), SubPolicy(0.5, "autocontrast", 8, 0.9, "equalize", 2, fillcolor), SubPolicy(0.2, "shearY", 7, 0.3, "posterize", 7, fillcolor), SubPolicy(0.4, "color", 3, 0.6, "brightness", 7, fillcolor), SubPolicy(0.3, "sharpness", 9, 0.7, "brightness", 9, fillcolor), SubPolicy(0.6, "equalize", 5, 0.5, "equalize", 1, fillcolor), SubPolicy(0.6, "contrast", 7, 0.6, "sharpness", 5, fillcolor), SubPolicy(0.7, "color", 7, 0.5, "translateX", 8, fillcolor), SubPolicy(0.3, "equalize", 7, 0.4, "autocontrast", 8, fillcolor), SubPolicy(0.4, "translateY", 3, 0.2, "sharpness", 6, fillcolor), SubPolicy(0.9, "brightness", 6, 0.2, "color", 8, fillcolor), SubPolicy(0.5, "solarize", 2, 0.0, "invert", 3, fillcolor), SubPolicy(0.2, "equalize", 0, 0.6, "autocontrast", 0, fillcolor), SubPolicy(0.2, "equalize", 8, 0.8, "equalize", 4, fillcolor), SubPolicy(0.9, "color", 9, 0.6, "equalize", 6, fillcolor), SubPolicy(0.8, "autocontrast", 4, 0.2, "solarize", 8, fillcolor), SubPolicy(0.1, "brightness", 3, 0.7, "color", 0, fillcolor), SubPolicy(0.4, "solarize", 5, 0.9, "autocontrast", 3, fillcolor), SubPolicy(0.9, "translateY", 9, 0.7, "translateY", 9, fillcolor), SubPolicy(0.9, "autocontrast", 2, 0.8, "solarize", 3, fillcolor), SubPolicy(0.8, "equalize", 8, 0.1, "invert", 3, fillcolor), SubPolicy(0.7, "translateY", 9, 0.9, "autocontrast", 1, fillcolor) ] def __call__(self, img): policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment CIFAR10 Policy" class SubPolicy(object): def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)): ranges = { "shearX": np.linspace(0, 0.3, 10), "shearY": np.linspace(0, 0.3, 10), "translateX": np.linspace(0, 150 / 331, 10), "translateY": np.linspace(0, 150 / 331, 10), "rotate": np.linspace(0, 30, 10), "color": np.linspace(0.0, 0.9, 10), "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int), "solarize": np.linspace(256, 0, 10), "contrast": np.linspace(0.0, 0.9, 10), "sharpness": np.linspace(0.0, 0.9, 10), "brightness": np.linspace(0.0, 0.9, 10), "autocontrast": [0] * 10, "equalize": [0] * 10, "invert": [0] * 10 } # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand def rotate_with_fill(img, magnitude): rot = img.convert("RGBA").rotate(magnitude) return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode) func = { "shearX": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), Image.BICUBIC, fillcolor=fillcolor), "shearY": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), Image.BICUBIC, fillcolor=fillcolor), "translateX": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0), fillcolor=fillcolor), "translateY": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])), fillcolor=fillcolor), "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude), # "rotate": lambda img, magnitude: img.rotate(magnitude * random.choice([-1, 1])), "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])), "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude), "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude), "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance( 1 + magnitude * random.choice([-1, 1])), "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance( 1 + magnitude * random.choice([-1, 1])), "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance( 1 + magnitude * random.choice([-1, 1])), "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img), "equalize": lambda img, magnitude: ImageOps.equalize(img), "invert": lambda img, magnitude: ImageOps.invert(img) } # self.name = "{}_{:.2f}_and_{}_{:.2f}".format( # operation1, ranges[operation1][magnitude_idx1], # operation2, ranges[operation2][magnitude_idx2]) self.p1 = p1 self.operation1 = func[operation1] self.magnitude1 = ranges[operation1][magnitude_idx1] self.p2 = p2 self.operation2 = func[operation2] self.magnitude2 = ranges[operation2][magnitude_idx2] def __call__(self, img): if random.random() < self.p1: img = self.operation1(img, self.magnitude1) if random.random() < self.p2: img = self.operation2(img, self.magnitude2) return img def seed_everything(seed): random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True SEED = 2019 seed_everything(SEED) use_cuda = cuda.is_available() print("use_cuda : {}".format(use_cuda)) class TrainDataset(Dataset): def __init__(self, df, mode='train', transforms=None): self.df = df self.mode = mode self.transform = transforms[self.mode] def __len__(self): return len(self.df) def __getitem__(self, idx): image = Image.open(TRAIN_IMAGE_PATH / self.df['img_file'][idx]).convert("RGB") if self.transform: image = self.transform(image) label = self.df['class'][idx] return image, label class TestDataset(Dataset): def __init__(self, df, mode='test', transforms=None): self.df = df self.mode = mode self.transform = transforms[self.mode] def __len__(self): return len(self.df) def __getitem__(self, idx): image = Image.open(TEST_IMAGE_PATH / self.df[idx]).convert("RGB") if self.transform: image = self.transform(image) return image def train_one_epoch(model, criterion, train_loader, optimizer, mixup_loss, accumulation_step=2): model.train() train_loss = 0. optimizer.zero_grad() for i, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.cuda(), targets.cuda() if mixup_loss: inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, alpha=1.0, use_cuda = use_cuda) # alpha in [0.4, 1.0] 선택 가능 inputs, targets_a, targets_b = map(Variable, (inputs, targets_a, targets_b)) outputs = model(inputs) loss = mixup_criterion(criterion, outputs.cuda(), targets_a.cuda(), targets_b.cuda(), lam) else: outputs = model(inputs) loss = criterion(outputs, targets) loss.backward() if accumulation_step: if (i+1) % accumulation_step == 0: optimizer.step() optimizer.zero_grad() else: optimizer.step() optimizer.zero_grad() train_loss += loss.item() / len(train_loader) return train_loss def validation(model, criterion, valid_loader, y_true): model.eval() valid_preds = np.zeros((len(valid_dataset), num_classes)) val_loss = 0. with torch.no_grad(): for i, (inputs, targets) in enumerate(valid_loader): inputs, targets = inputs.cuda(), targets.cuda() outputs = model(inputs).detach() loss = criterion(outputs, targets) valid_preds[i * batch_size: (i+1) * batch_size] = outputs.cpu().numpy() val_loss += loss.item() / len(valid_loader) y_pred = np.argmax(valid_preds, axis=1) val_score = f1_score(y_true, y_pred, average='micro') return val_loss, val_score def pick_best_score(result1, result2): if result1['best_score'] < result2['best_score']: return result2 else: return result1 def pick_best_loss(result1, result2): if result1['best_loss'] < result2['best_loss']: return result1 else: return result2 def train_model(num_epochs=60, accumulation_step=4, mixup_loss=False, cv_checkpoint=False, fine_tune=False, weight_file_name='weight_best.pt', **train_kwargs): # choose scheduler if opt == 'AdamW': #lr = 0.00001 lr = 1e-3 optimizer = AdamW(model.parameters(), lr=lr) mode = 'max' if cv_checkpoint else 'min' scheduler = ReduceLROnPlateau(optimizer, mode=mode, patience=10, factor=0.1) elif opt == 'SGDR': lr = 5e-3 optimizer = SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=0.025) eta_min = 1e-6 T_0 = 10 # 10 T_mult = 2 restart_decay = 1 scheduler = CosineAnnealingWarmRestarts(optimizer,T_0=T_0, eta_min=eta_min, T_mult=T_mult, restart_decay=restart_decay) train_result = {} train_result['weight_file_name'] = weight_file_name best_epoch = -1 best_score = 0. lrs = [] score = [] for epoch in range(num_epochs): start_time = time.time() train_loss = train_one_epoch(model, criterion, train_loader, optimizer, mixup_loss, accumulation_step) val_loss, val_score = validation(model, criterion, valid_loader, y_true) score.append(val_score) # model save (score or loss?) if cv_checkpoint: if val_score > best_score: best_score = val_score train_result['best_epoch'] = epoch + 1 train_result['best_score'] = round(best_score, 5) torch.save(model.state_dict(), weight_file_name) else: if val_loss < best_loss: best_loss = val_loss train_result['best_epoch'] = epoch + 1 train_result['best_loss'] = round(best_loss, 5) torch.save(model.state_dict(), weight_file_name) elapsed = time.time() - start_time lr = [_['lr'] for _ in optimizer.param_groups] print("Epoch {} - train_loss: {:.4f} val_loss: {:.4f} cv_score: {:.4f} lr: {:.6f} time: {:.0f}s b_epo: {} b_sco: {:.4f}".format( epoch+1, train_loss, val_loss, val_score, lr[0], elapsed, train_result['best_epoch'], train_result['best_score'])) for param_group in optimizer.param_groups: lrs.append(param_group['lr']) # scheduler update if opt == 'AdamW': if cv_checkpoint: scheduler.step(val_score) else: scheduler.step(val_loss) else: scheduler.step() return train_result, lrs, score which_fold = 0 batch_size = 48 num_epochs = 71 base_model = 'ResNext101_32x8d_wsl' # ResNet-50 ResNet-101(ResNext50) ResNext101 ResNet-101-re VGG19-bn ResNext101_32x8d_wsl mixup_loss = False opt = 'SGDR' # 'SGDR' #fold_file = "../input/car-fold-48/train_8folded.csv" fold_file = "../input/car-fold-48/train_4folded.csv" #fold_file = "../input/train-12folded/train_12folded.csv" print('use base model '+ base_model + ' for training') model, target_size = get_base_model(base_model, 196) fine_tune = False model.cuda() data_transforms = { 'train': vision.transforms.Compose([ vision.transforms.Resize(target_size), vision.transforms.RandomHorizontalFlip(), vision.transforms.RandomRotation(20), CIFAR10Policy(), #ImageNetPolicy(), vision.transforms.ToTensor(), vision.transforms.Normalize( [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'valid': vision.transforms.Compose([ vision.transforms.Resize(target_size), #vision.transforms.RandomResizedCrop(target_size, scale=(0.8,1.0)), #vision.transforms.RandomHorizontalFlip(), vision.transforms.ToTensor(), vision.transforms.Normalize( [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'test': vision.transforms.Compose([ vision.transforms.Resize(target_size), #0727 vision.transforms.RandomResizedCrop(target_size, scale=(0.8,1.0)), vision.transforms.ToTensor(), vision.transforms.Normalize( [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), } print(os.listdir('../input')) TRAIN_IMAGE_PATH = Path('../input/3rd-ml-month-car-image-cropping-dataset/train_crop/') TEST_IMAGE_PATH = Path('../input/3rd-ml-month-car-image-cropping-dataset/test_crop/') df = pd.read_csv(fold_file) test_csv = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/test.csv') df.head() print(len(df[df['fold'] == 0]), len(df[df['fold'] == 1]), len(df[df['fold'] == 2]), len(df[df['fold'] == 3])) criterion = nn.CrossEntropyLoss() train_df = df.loc[df['fold'] != which_fold] valid_df = df.loc[df['fold'] == which_fold] train_df = train_df[['img_file', 'class']].reset_index(drop=True) valid_df = valid_df[['img_file', 'class']].reset_index(drop=True) train_df.replace(196, 0, inplace=True) # 대회 데이터 클래스에 0이 없기에 일부러 바꿔줬습니다. model train시 클래스에 0이 없으면 오류 나기 때문에 valid_df.replace(196, 0, inplace=True) num_classes = train_df['class'].nunique() y_true = valid_df['class'].values print('{}th fold : n_train : {} n_valid : {}'.format(which_fold, len(train_df), len(valid_df))) train_dataset = TrainDataset(train_df, mode='train', transforms=data_transforms) valid_dataset = TrainDataset(valid_df, mode='valid', transforms=data_transforms) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False) train_kwargs = dict( train_loader=train_loader, valid_loader=valid_loader, model=model, criterion=criterion, y_true=y_true, opt=opt, ) print("training starts") weight_file_name = base_model + "_weight_best_{}.pt".format(which_fold) result, lrs, score = train_model(num_epochs=num_epochs, accumulation_step=2, mixup_loss=mixup_loss, cv_checkpoint=True, fine_tune=fine_tune, weight_file_name=weight_file_name, **train_kwargs) print(result)