{"cells":[{"metadata":{},"cell_type":"markdown","source":"+ 코드 조금 정리해서 single fold 기준으로 lb 0.9 이상 나오는 커널 공개합니다. (실전 대회 입문을 위한 baseline 입니다!)\n+ 이렇게 공개하는 이유는 대회 목적이 상품 보다는 공부 하는거에 더 의미가 있다고 생각하기 때문입니다. (상품에 목메시는 분은 없겠죠?)\n+ 그냥 fork해서 commit하는 일은 없도록 fold 나누는 csv 파일은 비공개로 해놨습니다.\n\n+ 7/8 update - batch accumulation 추가, learning rate scheduler 수정, lr과 score 시각화 추가, 코드 약간 수정\n+ 7/10 수정된 dataset 사용"},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"import os\nimport time\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport glob\nimport matplotlib.pyplot as plt\nfrom PIL import Image, ImageEnhance, ImageOps\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport torch\nfrom torch import nn, cuda\nfrom torch.autograd import Variable \nimport torch.nn.functional as F\nimport torchvision as vision\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam, SGD, Optimizer\nfrom torch.optim.lr_scheduler import _LRScheduler, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Useful stuffs"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class AdamW(Optimizer):\n    \"\"\"Implements AdamW algorithm.\n\n    It has been proposed in `Fixing Weight Decay Regularization in Adam`_.\n\n    Arguments:\n        params (iterable): iterable of parameters to optimize or dicts defining\n            parameter groups\n        lr (float, optional): learning rate (default: 1e-3)\n        betas (Tuple[float, float], optional): coefficients used for computing\n            running averages of gradient and its square (default: (0.9, 0.999))\n        eps (float, optional): term added to the denominator to improve\n            numerical stability (default: 1e-8)\n        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\n\n    .. Fixing Weight Decay Regularization in Adam:\n    https://arxiv.org/abs/1711.05101\n    \"\"\"\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\n                 weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps,\n                        weight_decay=weight_decay)\n        super(AdamW, self).__init__(params, defaults)\n\n    def step(self, closure=None):\n        \"\"\"Performs a single optimization step.\n\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                if grad.is_sparse:\n                    raise RuntimeError('AdamW does not support sparse gradients, please consider SparseAdam instead')\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state['step'] = 0\n                    # Exponential moving average of gradient values\n                    state['exp_avg'] = torch.zeros_like(p.data)\n                    # Exponential moving average of squared gradient values\n                    state['exp_avg_sq'] = torch.zeros_like(p.data)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                state['step'] += 1\n\n                # according to the paper, this penalty should come after the bias correction\n                # if group['weight_decay'] != 0:\n                #     grad = grad.add(group['weight_decay'], p.data)\n\n                # Decay the first and second moment running average coefficient\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n\n                denom = exp_avg_sq.sqrt().add_(group['eps'])\n\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n                step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1\n\n                p.data.addcdiv_(-step_size, exp_avg, denom)\n\n                if group['weight_decay'] != 0:\n                    p.data.add_(-group['weight_decay'], p.data)\n\n        return loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class CosineAnnealingWithRestartsLR(_LRScheduler):\n    '''\n    SGDR\\: Stochastic Gradient Descent with Warm Restarts: https://arxiv.org/abs/1608.03983\n    code: https://github.com/gurucharanmk/PyTorch_CosineAnnealingWithRestartsLR/blob/master/CosineAnnealingWithRestartsLR.py\n    added restart_decay value to decrease lr for every restarts\n    '''\n    def __init__(self, optimizer, T_max, eta_min=0, last_epoch=-1, T_mult=1, restart_decay=0.95):\n        self.T_max = T_max\n        self.T_mult = T_mult\n        self.next_restart = T_max\n        self.eta_min = eta_min\n        self.restarts = 0\n        self.last_restart = 0\n        self.T_num = 0\n        self.restart_decay = restart_decay\n        super(CosineAnnealingWithRestartsLR,self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        self.Tcur = self.last_epoch - self.last_restart\n        if self.Tcur >= self.next_restart:\n            self.next_restart *= self.T_mult\n            self.last_restart = self.last_epoch\n            self.T_num += 1\n        learning_rate = [(self.eta_min + ((base_lr)*self.restart_decay**self.T_num - self.eta_min) * (1 + math.cos(math.pi * self.Tcur / self.next_restart)) / 2) for base_lr in self.base_lrs]\n        return learning_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"'''\n유명한 mixup 논문. 타대회에서도 많이 씁니다.\nhttps://arxiv.org/abs/1710.09412\n'''\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\ndef mixup_data(x, y, alpha=1.0, use_cuda=True):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\n'''\nHomin님 kernel 참조\nAutoAugment: Learning Augmentation Policies from Data\nhttps://arxiv.org/abs/1905.\nCode: https://github.com/DeepVoltaire/AutoAugment\n'''\n    \nclass CIFAR10Policy(object):\n    \"\"\" Randomly choose one of the best 25 Sub-policies on CIFAR10.\n        Example:\n        >>> policy = CIFAR10Policy()\n        >>> transformed = policy(image)\n        Example as a PyTorch Transform:\n        >>> transform=transforms.Compose([\n        >>>     transforms.Resize(256),\n        >>>     CIFAR10Policy(),\n        >>>     transforms.ToTensor()])\n    \"\"\"\n    def __init__(self, fillcolor=(128, 128, 128)):\n        self.policies = [\n            SubPolicy(0.1, \"invert\", 7, 0.2, \"contrast\", 6, fillcolor),\n            SubPolicy(0.7, \"rotate\", 2, 0.3, \"translateX\", 9, fillcolor),\n            SubPolicy(0.8, \"sharpness\", 1, 0.9, \"sharpness\", 3, fillcolor),\n            SubPolicy(0.5, \"shearY\", 8, 0.7, \"translateY\", 9, fillcolor),\n            SubPolicy(0.5, \"autocontrast\", 8, 0.9, \"equalize\", 2, fillcolor),\n\n            SubPolicy(0.2, \"shearY\", 7, 0.3, \"posterize\", 7, fillcolor),\n            SubPolicy(0.4, \"color\", 3, 0.6, \"brightness\", 7, fillcolor),\n            SubPolicy(0.3, \"sharpness\", 9, 0.7, \"brightness\", 9, fillcolor),\n            SubPolicy(0.6, \"equalize\", 5, 0.5, \"equalize\", 1, fillcolor),\n            SubPolicy(0.6, \"contrast\", 7, 0.6, \"sharpness\", 5, fillcolor),\n\n            SubPolicy(0.7, \"color\", 7, 0.5, \"translateX\", 8, fillcolor),\n            SubPolicy(0.3, \"equalize\", 7, 0.4, \"autocontrast\", 8, fillcolor),\n            SubPolicy(0.4, \"translateY\", 3, 0.2, \"sharpness\", 6, fillcolor),\n            SubPolicy(0.9, \"brightness\", 6, 0.2, \"color\", 8, fillcolor),\n            SubPolicy(0.5, \"solarize\", 2, 0.0, \"invert\", 3, fillcolor),\n\n            SubPolicy(0.2, \"equalize\", 0, 0.6, \"autocontrast\", 0, fillcolor),\n            SubPolicy(0.2, \"equalize\", 8, 0.8, \"equalize\", 4, fillcolor),\n            SubPolicy(0.9, \"color\", 9, 0.6, \"equalize\", 6, fillcolor),\n            SubPolicy(0.8, \"autocontrast\", 4, 0.2, \"solarize\", 8, fillcolor),\n            SubPolicy(0.1, \"brightness\", 3, 0.7, \"color\", 0, fillcolor),\n\n            SubPolicy(0.4, \"solarize\", 5, 0.9, \"autocontrast\", 3, fillcolor),\n            SubPolicy(0.9, \"translateY\", 9, 0.7, \"translateY\", 9, fillcolor),\n            SubPolicy(0.9, \"autocontrast\", 2, 0.8, \"solarize\", 3, fillcolor),\n            SubPolicy(0.8, \"equalize\", 8, 0.1, \"invert\", 3, fillcolor),\n            SubPolicy(0.7, \"translateY\", 9, 0.9, \"autocontrast\", 1, fillcolor)\n        ]\n\n\n    def __call__(self, img):\n        policy_idx = random.randint(0, len(self.policies) - 1)\n        return self.policies[policy_idx](img)\n\n    def __repr__(self):\n        return \"AutoAugment CIFAR10 Policy\"\n\n    \nclass SubPolicy(object):\n    def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)):\n        ranges = {\n            \"shearX\": np.linspace(0, 0.3, 10),\n            \"shearY\": np.linspace(0, 0.3, 10),\n            \"translateX\": np.linspace(0, 150 / 331, 10),\n            \"translateY\": np.linspace(0, 150 / 331, 10),\n            \"rotate\": np.linspace(0, 30, 10),\n            \"color\": np.linspace(0.0, 0.9, 10),\n            \"posterize\": np.round(np.linspace(8, 4, 10), 0).astype(np.int),\n            \"solarize\": np.linspace(256, 0, 10),\n            \"contrast\": np.linspace(0.0, 0.9, 10),\n            \"sharpness\": np.linspace(0.0, 0.9, 10),\n            \"brightness\": np.linspace(0.0, 0.9, 10),\n            \"autocontrast\": [0] * 10,\n            \"equalize\": [0] * 10,\n            \"invert\": [0] * 10\n        }\n\n        # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand\n        def rotate_with_fill(img, magnitude):\n            rot = img.convert(\"RGBA\").rotate(magnitude)\n            return Image.composite(rot, Image.new(\"RGBA\", rot.size, (128,) * 4), rot).convert(img.mode)\n\n        func = {\n            \"shearX\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0),\n                Image.BICUBIC, fillcolor=fillcolor),\n            \"shearY\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0),\n                Image.BICUBIC, fillcolor=fillcolor),\n            \"translateX\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0),\n                fillcolor=fillcolor),\n            \"translateY\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])),\n                fillcolor=fillcolor),\n            \"rotate\": lambda img, magnitude: rotate_with_fill(img, magnitude),\n            # \"rotate\": lambda img, magnitude: img.rotate(magnitude * random.choice([-1, 1])),\n            \"color\": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])),\n            \"posterize\": lambda img, magnitude: ImageOps.posterize(img, magnitude),\n            \"solarize\": lambda img, magnitude: ImageOps.solarize(img, magnitude),\n            \"contrast\": lambda img, magnitude: ImageEnhance.Contrast(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"sharpness\": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"brightness\": lambda img, magnitude: ImageEnhance.Brightness(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"autocontrast\": lambda img, magnitude: ImageOps.autocontrast(img),\n            \"equalize\": lambda img, magnitude: ImageOps.equalize(img),\n            \"invert\": lambda img, magnitude: ImageOps.invert(img)\n        }\n\n        # self.name = \"{}_{:.2f}_and_{}_{:.2f}\".format(\n        #     operation1, ranges[operation1][magnitude_idx1],\n        #     operation2, ranges[operation2][magnitude_idx2])\n        self.p1 = p1\n        self.operation1 = func[operation1]\n        self.magnitude1 = ranges[operation1][magnitude_idx1]\n        self.p2 = p2\n        self.operation2 = func[operation2]\n        self.magnitude2 = ranges[operation2][magnitude_idx2]\n\n\n    def __call__(self, img):\n        if random.random() < self.p1: img = self.operation1(img, self.magnitude1)\n        if random.random() < self.p2: img = self.operation2(img, self.magnitude2)\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# seed value fix\n# seed 값을 고정해야 hyper parameter 바꿀 때마다 결과를 비교할 수 있습니다.\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 2019\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_cuda = cuda.is_available()\nuse_cuda","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, df, mode='train', transforms=None):\n        self.df = df\n        self.mode = mode\n        self.transform = transforms[self.mode]\n        \n    def __len__(self):\n        return len(self.df)\n            \n    def __getitem__(self, idx):\n        \n        image = Image.open(TRAIN_IMAGE_PATH / self.df['img_file'][idx]).convert(\"RGB\")\n\n        if self.transform:\n            image = self.transform(image)\n\n        label = self.df['class'][idx]\n\n        return image, label\n\n    \nclass TestDataset(Dataset):\n    def __init__(self, df, mode='test', transforms=None):\n        self.df = df\n        self.mode = mode\n        self.transform = transforms[self.mode]\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        image = Image.open(TEST_IMAGE_PATH / self.df[idx]).convert(\"RGB\")\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return image        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_size = (224, 224)\n\ndata_transforms = {\n    'train': vision.transforms.Compose([\n        vision.transforms.Resize(target_size),\n        vision.transforms.RandomHorizontalFlip(),\n        vision.transforms.RandomRotation(20),\n        CIFAR10Policy(),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n    'valid': vision.transforms.Compose([\n        vision.transforms.Resize(target_size),\n        vision.transforms.RandomResizedCrop(target_size, scale=(0.8,1.0)),\n        vision.transforms.RandomHorizontalFlip(),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n    'test': vision.transforms.Compose([\n        vision.transforms.Resize((224,224)),\n        vision.transforms.RandomResizedCrop(target_size, scale=(0.8,1.0)),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ncrop된 이미지 사용\nreference 허태명님 커널: https://www.kaggle.com/tmheo74/3rd-ml-month-car-image-cropping\n'''\n\nTRAIN_IMAGE_PATH = Path('../input/kakl-3rd-cropped-dataset/train_crop/')\nTEST_IMAGE_PATH = Path('../input/kakl-3rd-cropped-dataset/test_crop/')\n# train_image_path = Path('../input/2019-3rd-ml-month-with-kakr/train/')\n# test_image_path = Path('../input/2019-3rd-ml-month-with-kakr/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 미리 5 fold로 나누어 csv로 저장한 후 불러왔습니다.\n# 80프로를 train set으로, 나머지 20프로를 validation set으로 사용합니다. => 수정: 실수로 4 kfold를 해버렸네요 (3/4 train set, 1/4 valid set입니다)\ndf = pd.read_csv(\"../input/car-folds/car_4folds.csv\")\ntest_csv = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/test.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class 분포 고려하여 사전에 split 해놨습니다. fold별 개수 확인 가능\nlen(df[df['fold'] == 0]), len(df[df['fold'] == 1]), len(df[df['fold'] == 2]), len(df[df['fold'] == 3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = df.loc[df['fold'] != 0]\nvalid_df = df.loc[df['fold'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for debugging\ntrain_df = train_df[:900]\nvalid_df = valid_df[:900]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ddtrain_df['class'] = train_df['class'] + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[['img_file', 'class']].reset_index(drop=True)\nvalid_df = valid_df[['img_file', 'class']].reset_index(drop=True)\nx_test = test_csv['img_file']\ntrain_df.replace(196, 0, inplace=True) # 대회 데이터 클래스에 0이 없기에 일부러 바꿔줬습니다. model train시 클래스에 0이 없으면 오류 나기 때문에\n\nnum_classes = train_df['class'].nunique()\ny_true = valid_df['class'].values # for cv score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"number of train dataset: {}\".format(len(train_df)))\nprint(\"number of valid dataset: {}\".format(len(valid_df)))\nprint(\"number of classes to predict: {}\".format(num_classes))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, criterion, train_loader, optimizer, mixup_loss, accumulation_step=2):\n    \n    model.train()\n    train_loss = 0.\n    optimizer.zero_grad()\n\n    for i, (inputs, targets) in enumerate(train_loader):\n            \n        inputs, targets = inputs.cuda(), targets.cuda()\n\n        if mixup_loss:\n            inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, alpha=1.0, use_cuda = use_cuda) # alpha in [0.4, 1.0] 선택 가능\n            inputs, targets_a, targets_b = map(Variable, (inputs, targets_a, targets_b))\n            outputs = model(inputs)\n            loss = mixup_criterion(criterion, outputs.cuda(), targets_a.cuda(), targets_b.cuda(), lam)\n            \n        else:\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n\n        loss.backward()\n        \n        if accumulation_step:\n            if (i+1) % accumulation_step == 0:  \n                optimizer.step()\n                optimizer.zero_grad()\n        else:\n            optimizer.step()\n            optimizer.zero_grad()\n        \n\n        train_loss += loss.item() / len(train_loader)\n        \n    return train_loss\n\n\ndef validation(model, criterion, valid_loader):\n    \n    model.eval()\n    valid_preds = np.zeros((len(valid_dataset), num_classes))\n    val_loss = 0.\n    \n    with torch.no_grad():\n        for i, (inputs, targets) in enumerate(valid_loader):\n\n            inputs, targets = inputs.cuda(), targets.cuda()\n            \n            outputs = model(inputs).detach()\n            loss = criterion(outputs, targets)\n            valid_preds[i * batch_size: (i+1) * batch_size] = outputs.cpu().numpy()\n            \n            val_loss += loss.item() / len(valid_loader)\n            \n        y_pred = np.argmax(valid_preds, axis=1)\n        val_score = f1_score(y_true, y_pred, average='micro')  \n        \n    return val_loss, val_score    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 스코어 기준과 loss 기준. lb 점수가 cv score와 비교했을 때 굉장히\n# consistent해서 cv score를 기준으로 합니다.\ndef pick_best_score(result1, result2):\n    if result1['best_score'] < result2['best_score']:\n        return result2\n    else:\n        return result1\n    \ndef pick_best_loss(result1, result2):\n    if result1['best_loss'] < result2['best_loss']:\n        return result1\n    else:\n        return result2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(num_epochs=60, accumulation_step=4, mixup_loss=False, cv_checkpoint=False, fine_tune=False, weight_file_name='weight_best.pt', **train_kwargs):\n    \n    # choose scheduler\n    if fine_tune:\n        lr = 0.00001\n        optimizer = AdamW(model.parameters(), lr=lr, weight_decay=0.000025)   \n        scheduler = ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1)\n    else:    \n        lr = 0.01\n        optimizer = SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=0.025)\n        eta_min = 1e-6\n        T_max = 10\n        T_mult = 1\n        restart_decay = 0.97\n        scheduler = CosineAnnealingWithRestartsLR(optimizer,T_max=T_max, eta_min=eta_min, T_mult=T_mult, restart_decay=restart_decay)\n\n    train_result = {}\n    train_result['weight_file_name'] = weight_file_name\n    best_epoch = -1\n    best_score = 0.\n    lrs = []\n    score = []\n    \n    for epoch in range(num_epochs):\n        \n        start_time = time.time()\n\n        train_loss = train_one_epoch(model, criterion, train_loader, optimizer, mixup_loss, accumulation_step)\n        val_loss, val_score = validation(model, criterion, valid_loader)\n        score.append(val_score)\n    \n        # model save (score or loss?)\n        if cv_checkpoint:\n            if val_score > best_score:\n                best_score = val_score\n                train_result['best_epoch'] = epoch + 1\n                train_result['best_score'] = round(best_score, 5)\n                torch.save(model.state_dict(), weight_file_name)\n        else:\n            if val_loss < best_loss:\n                best_loss = val_loss\n                train_result['best_epoch'] = epoch + 1\n                train_result['best_loss'] = round(best_loss, 5)\n                torch.save(model.state_dict(), weight_file_name)\n        \n        elapsed = time.time() - start_time\n        \n        lr = [_['lr'] for _ in optimizer.param_groups]\n        print(\"Epoch {} - train_loss: {:.4f}  val_loss: {:.4f}  cv_score: {:.4f}  lr: {:.6f}  time: {:.0f}s\".format(\n                epoch+1, train_loss, val_loss, val_score, lr[0], elapsed))\n        \n        for param_group in optimizer.param_groups:\n            lrs.append(param_group['lr'])\n        \n        # scheduler update\n        if fine_tune:\n            if cv_checkpoint:\n                scheduler.step(val_score)\n            else:\n                scheduler.step(val_loss)\n        else:\n            scheduler.step()\n     \n    return train_result, lrs, score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_dataset = TrainDataset(train_df, mode='train', transforms=data_transforms)\nvalid_dataset = TrainDataset(valid_df, mode='valid', transforms=data_transforms)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# baseline이기 때문에 resnet50 사용합니다. 바꿔보세요!\nmodel = models.resnet50(pretrained=True)\nmodel.fc = nn.Linear(2048, num_classes)\nmodel.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\ntrain_kwargs = dict(\n    train_loader=train_loader,\n    valid_loader=valid_loader,\n    model=model,\n    criterion=criterion,\n    )\n\n\nprint(\"training starts\")\nnum_epochs = 120\nresult, lrs, score = train_model(num_epochs=num_epochs, accumulation_step=2, mixup_loss=False, cv_checkpoint=True, fine_tune=False, weight_file_name='weight_best.pt', **train_kwargs)\nprint(result)\n\n\n# finetuning 부분은 전 버전 참고하시면 좋을것 같습니다.","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# learning rate plot\nplt.figure(figsize=(18,4))\nplt.subplot(1,2,1)\nplt.plot(lrs, 'b')\nplt.xlabel('Epochs', fontsize=12, fontweight='bold')\nplt.ylabel('Learning rate', fontsize=14, fontweight='bold')\nplt.title('Learning rate schedule', fontsize=15, fontweight='bold')\n\nx = [x for x in range(0, num_epochs, 10)]\ny = [0.01, 0.005, 0.000001]\nylabel = ['1e-2', '1e-4', '1e-6']\nplt.xticks(x)\nplt.yticks(y, ylabel)\n\nplt.subplot(1,2,2)\nplt.plot(score, 'r')\nplt.xlabel('Epochs', fontsize=12, fontweight='bold')\nplt.ylabel('Valid score', fontsize=14, fontweight='bold')\nplt.title('F1 Score', fontsize=15, fontweight='bold')\n\nx = [x for x in range(0, num_epochs, 10)]\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 저장한 weight 불러와서 predict  \n# 최근에 열린 imet 대회 같은 경우는 학습 시간이 9시간 이상 해야하기 때문에 저장하고 불러오기가 중요합니다\n# 보통 kaggle에서 딥러닝 대회는 training과 inference는 따로 커널을 만들어서 진행합니다 (저처럼 local gpu 없을 경우 필수)\n\nmodel = models.resnet50() \nmodel.fc = nn.Linear(2048, num_classes)\nmodel.cuda()\nmodel.load_state_dict(torch.load(result['weight_file_name']))\n\nbatch_size = 1 # 배치 1로 주면 순서대로 나온다\ntest_dataset = TestDataset(x_test, mode='test', transforms=data_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\nmodel.eval()\ntest_preds = []\n\nwith torch.no_grad():\n    for i, images in enumerate(tqdm_notebook(test_loader)):\n        images = images.cuda()\n    \n        preds = model(images).detach()\n        test_preds.append(preds.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs = []\nfor _ in test_preds:\n    # argmax를 사용해서 가장 높은 확률로 예측한 class 반환\n    predicted_class_indices=np.argmax(_, axis=1).tolist()\n    outputs.append(predicted_class_indices)\n\nresult = np.concatenate(outputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/sample_submission.csv')\nsubmission[\"class\"] = result\nsubmission[\"class\"].replace(0, 196, inplace=True) # 196에서 0으로 수정했던걸 다시 되돌려준다 \nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## To do and Reference"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"'''\n참고하면 좋을 논문들. 타 대회에서도 다 적용 가능합니다.\n\nSnapshot Ensembles: Train 1, get M for free - 심심해서 해봤는데 조금 오르긴 합니다.\nhttps://arxiv.org/abs/1704.00109 \n\nSGDR: Stochastic Gradient Descent with Warm Restarts\nhttps://arxiv.org/abs/1608.03983\n\nUnsupervised Data Augmentation - 김일두님 깃헙 참고해서 구현해보면 좋을것 같습니다\nhttps://arxiv.org/abs/1904.12848\n\nAutoAugment: Learning Augmentation Policies from Data\nhttps://arxiv.org/abs/1805.09501\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"+ 코드 잘못된 부분 지적 부탁드립니다.\n+ **도움 되셨으면 우측 상단에 Upvote 부탁드립니다!**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}