{"cells":[{"metadata":{},"cell_type":"markdown","source":"ResNet18 + ArcFace"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorwatch\n!pip install graphviz\n!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom PIL import Image, ImageFilter\nimport os\nimport numpy as np\nimport math\nimport time\nimport random\nimport pandas as pd\n\nimport torch\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport torchvision\nimport torch.nn as nn\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nimport torch.backends.cudnn as cudnn\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.nn import Parameter\n\nfrom torchvision import transforms\nimport torchvision.models as models\n\nimport pretrainedmodels\n\nimport warnings\nwarnings.filterwarnings(action='ignore')\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train set:')\nfor cls in os.listdir('../input/train/train'):\n print('{}:{}'.format(cls, len(os.listdir(os.path.join('../input/train/train', cls)))))\nim = Image.open('../input/train/train/cgm/train-cgm-738.jpg')\nprint(im.size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# HP"},{"metadata":{"trusted":true},"cell_type":"code","source":"Model = 'resnet18' # se_resnext50_32x4d || resnet18\nCheckpoint = 'resnet18'\nLoss = 'LabelSmoothSoftmaxCE' # FocalLoss || CrossEntropy || LabelSmoothSoftmaxCE\n\nFreeze = True\nResume = False\n\nNum_classes = 5\nSize = 224 # image size\nBatch_size = 256\nNum_epochs = 300\nInit_lr = 0.0001 \nStep_size = 20 # StepLr decay rate\n\n# warmup lr schedule\nMultiplier = 80\nTotal_epoch = 20\n\n# Focal Loss\nAlpha = 0.25 # 0.3\nGamma = 1.5 # 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed everything"},{"metadata":{"trusted":true},"cell_type":"code","source":"def setup_seed(seed):\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n np.random.seed(seed)\n random.seed(seed)\n torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data agumentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def RandomErasing(im, probability = 0.5, sl = 0.02, sh = 0.4, r1 = 0.3, mean=[128, 128, 128]):\n '''\n performs Random Erasing in Random Erasing Data Augmentation by Zhong et al. \n -------------------------------------------------------------------------------------\n img: PIL img\n probability: The probability that the operation will be performed.\n sl: min erasing area\n sh: max erasing area\n r1: min aspect ratio\n mean: erasing value\n -------------------------------------------------------------------------------------\n '''\n if random.uniform(0, 1) > probability:\n return im\n\n else:\n img = np.array(im)\n area = img.shape[0] * img.shape[1]\n \n while True:\n target_area = random.uniform(sl, sh) * area\n aspect_ratio = random.uniform(r1, 1/r1)\n h = int(round(math.sqrt(target_area * aspect_ratio)))\n w = int(round(math.sqrt(target_area / aspect_ratio)))\n if img.shape[0] > w and img.shape[1] > h:\n break\n\n if w < img.shape[0] and h < img.shape[1]:\n x1 = random.randint(0, img.shape[0] - w)\n y1 = random.randint(0, img.shape[1] - h)\n # if img.size()[0] == 3:\n if im.mode == 'RGB':\n img[x1:x1+h, y1:y1+w, 0] = mean[0]\n img[x1:x1+h, y1:y1+w, 1] = mean[1]\n img[x1:x1+h, y1:y1+w, 2] = mean[2]\n elif im.mode == 'L':\n img[x1:x1+h, y1:y1+w] = mean[0]\n img = Image.fromarray(np.uint8(img))\n\n return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train\ncls2label = {}\nfor label, cls in enumerate(os.listdir('../input/train/train')):\n cls2label[cls] = label\nprint(cls2label)\n\nims2labels = {}\nims2labels_train = {}\nims2labels_val = {}\nfor cls in os.listdir('../input/train/train'):\n im_num = len(os.listdir(os.path.join('../input/train/train', cls)))\n # total ims\n for im in os.listdir(os.path.join('../input/train/train', cls)):\n impath = os.path.join('../input/train/train', cls, im)\n ims2labels[impath] = cls2label[cls]\n val_ims = random.sample(os.listdir(os.path.join('../input/train/train', cls)), int(im_num*0.1))\n for im in val_ims:\n impath = os.path.join('../input/train/train', cls, im)\n ims2labels_val[impath] = cls2label[cls]\n for im in os.listdir(os.path.join('../input/train/train', cls)):\n if im not in val_ims:\n impath = os.path.join('../input/train/train', cls, im)\n ims2labels_train[impath] = cls2label[cls]\n \nprint('total:', list(ims2labels.items())[:5], len(list(ims2labels.items())))\nprint('train:', list(ims2labels_train.items())[:5], len(list(ims2labels_train.items())))\nprint('validation:', list(ims2labels_val.items())[:5], len(list(ims2labels_val.items())))\n\n# test\ndf_test = pd.read_csv('../input/sample_submission_file.csv')\ntest_data = df_test['Id']\n\nclass CDCDataset(Dataset):\n def __init__(self, dataset, transform=None, mode='train', tta=False, idx=0):\n self.tta=tta\n self.idx = idx\n self.mode = mode\n if self.mode == 'train' or self.mode == 'val':\n self.ims, self.labels = [], []\n for item in dataset.items():\n self.ims.append(item[0])\n self.labels.append(item[1])\n # print(self.ims, self.labels)\n elif self.mode == 'test':\n self.im_names = dataset\n self.ims = [os.path.join('../input/test/test/0', im) for im in dataset]\n self.transform = transform\n\n def __getitem__(self, index):\n im_path = self.ims[index]\n if self.mode == 'train' or self.mode == 'val':\n label = self.labels[index]\n elif self.mode == 'test':\n im_name = self.im_names[index]\n im = Image.open(im_path)\n if self.mode == 'train' or self.mode == 'val':\n if self.transform is not None:\n im = RandomErasing(im, probability = 0.5, sl = 0.02, sh = 0.4, r1 = 0.3, mean=[128, 128, 128])\n im = self.transform(im)\n return im, label\n elif self.mode == 'test':\n if self.tta:\n w, h = im.size\n if self.idx == 0:\n im = im.crop((0, 0, int(w*0.9), int(h*0.9))) # top left\n elif self.idx == 1:\n im = im.crop((int(w*0.1), 0, w, int(h*0.9))) # top right\n elif self.idx == 2:\n im = im.crop((int(w*0.05), int(h*0.05), w-int(w*0.05), h-int(h*0.05))) # center\n elif self.idx == 3:\n im = im.crop((0, int(h*0.1), w-int(w*0.1), h)) # bottom left\n elif self.idx == 4:\n im = im.crop((int(w*0.1), int(h*0.1), w, h)) # bottom right\n elif self.idx == 5:\n im = im.crop((0, 0, int(w*0.9), int(h*0.9))) \n im = im.transpose(Image.FLIP_LEFT_RIGHT) # top left and HFlip\n elif self.idx == 6:\n im = im.crop((int(w*0.1), 0, w, int(h*0.9)))\n im = im.transpose(Image.FLIP_LEFT_RIGHT) # top right and HFlip\n elif self.idx == 7:\n im = im.crop((int(w*0.05), int(h*0.05), w-int(w*0.05), h-int(h*0.05)))\n im = im.transpose(Image.FLIP_LEFT_RIGHT) # center and HFlip\n elif self.idx == 8:\n im = im.crop((0, int(h*0.1), w-int(w*0.1), h))\n im = im.transpose(Image.FLIP_LEFT_RIGHT) # bottom left and HFlip\n elif self.idx == 9:\n im = im.crop((int(w*0.1), int(h*0.1), w, h))\n im = im.transpose(Image.FLIP_LEFT_RIGHT) # bottom right and HFlip\n if self.transform is not None:\n im = self.transform(im)\n return im, im_name\n\n def __len__(self):\n return len(self.ims)\n\nif __name__ == '__main__':\n transform = transforms.Compose([transforms.ToTensor()])\n dst_train = CDCDataset(ims2labels, transform=transform)\n dataloader_train = DataLoader(dst_train, shuffle=True, batch_size=1, num_workers=0)\n #for im, loc, cls in dataloader_train:\n for data in dataloader_train:\n print(data)\n break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model\n* resnet18\n* se_resnext50"},{"metadata":{"trusted":true},"cell_type":"code","source":"def whitening(im):\n batch_size, channel, h, w = im.shape\n mean = [0.485, 0.456, 0.406]\n std = [0.229, 0.224, 0.225]\n im = torch.cat([(im[:,[0]]-mean[0])/std[0],\n (im[:,[1]]-mean[1])/std[1],\n (im[:,[2]]-mean[2])/std[2]], 1)\n return im\n\ndef l2_norm(x):\n norm = torch.norm(x, p=2, dim=1, keepdim=True)\n x = torch.div(x, norm)\n return x\n\nclass ResNet18(nn.Module):\n def __init__(self, model, num_classes=1000):\n super(ResNet18, self).__init__()\n self.backbone = model\n\n #self.fc1 = nn.Linear(512, 128)\n self.dropout = nn.Dropout(0.5)\n #self.fc2 = nn.Linear(128, num_classes)\n self.fc = nn.Linear(512, num_classes)\n \n self.conv_last = nn.Conv2d(512, num_classes, 1)\n \n\n def forward(self, x):\n # x = whitening(x)\n x = self.backbone.conv1(x)\n x = self.backbone.bn1(x)\n x = self.backbone.relu(x)\n x = self.backbone.maxpool(x)\n\n x = self.backbone.layer1(x)\n x = self.backbone.layer2(x)\n x = self.backbone.layer3(x)\n x = self.backbone.layer4(x)\n\n # print(x.size())\n x = self.backbone.avgpool(x)\n \n '''\n # FC\n x = x.view(x.size(0), -1)\n x = l2_norm(x)\n x = self.dropout(x) \n x = self.fc(x)\n x = l2_norm(x)\n '''\n \n # Full conv\n #x = self.conv_last(x)\n x = x.view(x.size(0), -1)\n x = l2_norm(x)\n\n return x\n \n#if __name__ == '__main__':\n# backbone = models.resnet18(pretrained=True)\n# models = ResNet18(backbone, 5)\n# data = torch.randn(1, 3, 224, 224)\n# x = models(data)\n# #print(x)\n# print(x.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class se_resnext50_32x4d(nn.Module):\n def __init__(self, model, num_classes=1000):\n super(se_resnext50_32x4d, self).__init__()\n self.backbone = model\n\n #self.fc1 = nn.Linear(512, 128)\n self.dropout = nn.Dropout(0.5)\n #self.fc2 = nn.Linear(128, num_classes)\n self.fc = nn.Linear(2048, 1024)\n \n self.conv_last = nn.Conv2d(2048, num_classes, 3)\n\n def forward(self, x):\n # x = whitening(x)\n x = self.backbone.layer0(x)\n x = self.backbone.layer1(x)\n x = self.backbone.layer2(x)\n x = self.backbone.layer3(x)\n x = self.backbone.layer4(x)\n x = self.backbone.avg_pool(x)\n\n # FC\n x = x.view(x.size(0), -1)\n x = self.fc(x)\n \n# # Full conv\n# x = self.conv_last(x)\n# x = x.view(x.size(0), -1)\n \n x = l2_norm(x)\n\n return x\n# if __name__ == '__main__':\n# backbone = pretrainedmodels.se_resnext50_32x4d(pretrained='imagenet')\n# models = se_resnext50_32x4d(backbone, 5)\n# data = torch.randn(1, 3, 224, 224)\n# x = models(data)\n# #print(x)\n# print(x.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Loss\n* Focal Loss\n* ArcFace"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n r\"\"\"\n This criterion is a implemenation of Focal Loss, which is proposed in \n Focal Loss for Dense Object Detection.\n Loss(x, class) = - \\alpha (1-softmax(x)[class])^gamma \\log(softmax(x)[class])\n The losses are averaged across observations for each minibatch.\n Args:\n alpha(1D Tensor, Variable) : the scalar factor for this criterion\n gamma(float, double) : gamma > 0; reduces the relative loss for well-classified examples (p > .5), \n putting more focus on hard, misclassified examples\n size_average(bool): By default, the losses are averaged over observations for each minibatch.\n However, if the field size_average is set to False, the losses are\n instead summed for each minibatch.\n \"\"\"\n def __init__(self, class_num, alpha=None, gamma=2, size_average=True):\n super(FocalLoss, self).__init__()\n if alpha is None:\n self.alpha = Variable(torch.ones(class_num, 1))\n else:\n if isinstance(alpha, Variable):\n self.alpha = torch.ones(class_num, 1)*alpha\n else:\n # self.alpha = Variable(alpha).cuda()\n self.alpha = Variable(torch.ones(class_num, 1)*alpha).cuda()\n self.gamma = gamma\n self.class_num = class_num\n self.size_average = size_average\n\n def forward(self, inputs, targets):\n N = inputs.size(0)\n C = inputs.size(1)\n P = F.softmax(inputs)\n\n class_mask = inputs.data.new(N, C).fill_(0)\n class_mask = Variable(class_mask)\n ids = targets.view(-1, 1)\n class_mask.scatter_(1, ids.data, 1.)\n\n if inputs.is_cuda and not self.alpha.is_cuda:\n self.alpha = self.alpha.cuda()\n alpha = self.alpha[ids.data.view(-1)]\n\n probs = (P*class_mask).sum(1).view(-1,1)\n\n log_p = probs.log()\n\n batch_loss = -alpha*(torch.pow((1-probs), self.gamma))*log_p \n\n if self.size_average:\n loss = batch_loss.mean()\n else:\n loss = batch_loss.sum()\n return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n r\"\"\"Implement of large margin arc distance:\n Args:\n in_features: size of each input sample\n out_features: size of each output sample\n s: norm of input feature\n m: margin\n cos(theta + m)\n \"\"\"\n def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n super(ArcMarginProduct, self).__init__()\n self.in_features = in_features\n self.out_features = out_features\n self.s = s\n self.m = m\n self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n nn.init.xavier_uniform_(self.weight)\n\n self.easy_margin = easy_margin\n self.cos_m = math.cos(m)\n self.sin_m = math.sin(m)\n self.th = math.cos(math.pi - m)\n self.mm = math.sin(math.pi - m) * m\n\n def forward(self, input, label):\n # --------------------------- cos(theta) & phi(theta) ---------------------------\n cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n phi = cosine * self.cos_m - sine * self.sin_m\n if self.easy_margin:\n phi = torch.where(cosine > 0, phi, cosine)\n else:\n phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n # --------------------------- convert label to one-hot ---------------------------\n # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n one_hot = torch.zeros(cosine.size(), device='cuda')\n one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n output = (one_hot * phi) + ((1.0 - one_hot) * cosine) # you can use torch.where if your torch.__version__ is 0.4\n output *= self.s\n # print(output)\n\n return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(output, target, topk=(1, 5)):\n \"\"\"Computes the precision@k for the specified values of k\"\"\"\n maxk = max(topk)\n batch_size = target.size(0)\n\n _, pred = output.topk(maxk, 1, True, True)\n pred = pred.t()\n correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n res = []\n for k in topk:\n correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n res.append(correct_k.mul_(100.0 / batch_size))\n return res","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# validation\ndef eval(model, dataloader_valid, criterion):\n sum = 0\n val_loss_sum = 0\n val_top1_sum = 0\n model.eval()\n for ims, label in dataloader_valid:\n input_val = Variable(ims).cuda()\n target_val = Variable(label).cuda()\n output_val = model(input_val)\n loss = criterion(output_val, target_val)\n top1_val = accuracy(output_val.data, target_val.data, topk=(1,))\n \n sum += 1\n val_loss_sum += loss.data.cpu().numpy()\n val_top1_sum += top1_val[0]\n avg_loss = val_loss_sum / sum\n avg_top1 = val_top1_sum / sum\n return avg_loss, avg_top1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# inference\ndef inference(model): \n # test data\n test_transform = transforms.Compose([transforms.Resize((int(Size), int(Size))),\n #transforms.TenCrop(Size),\n #Lambda(lambda crops: torch.stack([ToTensor()(crop) for crop in crops])),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225])])\n dst_test = CDCDataset(test_data, transform=test_transform, mode='test')\n dataloader_test = DataLoader(dst_test, shuffle=False, batch_size=Batch_size//2, num_workers=8)\n\n \n model.eval()\n results = []\n print('Inferencing ...')\n for ims, im_names in dataloader_test:\n input = Variable(ims).cuda()\n output = model(input)\n _, preds = output.topk(1, 1, True, True)\n preds = preds.cpu().detach().numpy()\n for pred, im_name in zip(preds, im_names):\n top1_name = [list(cls2label.keys())[list(cls2label.values()).index(p)] for p in pred]\n results.append({'Id':im_name, 'Category':''.join(top1_name)})\n df = pd.DataFrame(results, columns=['Category', 'Id'])\n df.to_csv('sub.csv', index=False)\ndef inference_TTA(model): \n # print(model)\n # test data\n test_transform = transforms.Compose([transforms.Resize((int(Size), int(Size))),\n #transforms.TenCrop(Size),\n #Lambda(lambda crops: torch.stack([ToTensor()(crop) for crop in crops])),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225])])\n # 10 TTA\n TTA10 = []\n results = []\n print('Inferencing with TTA ...')\n for idx in range(10):\n dst_test = CDCDataset(test_data, transform=test_transform, mode='test', tta=True, idx=idx)\n dataloader_test = DataLoader(dst_test, shuffle=False, batch_size=Batch_size//2, num_workers=0)\n\n\n model.eval().cuda()\n names2probs = {}\n for ims, im_names in dataloader_test:\n input = Variable(ims).cuda()\n output = model(input)\n probs = F.softmax(output)\n probs = probs.cpu().detach().numpy()\n for prob, im_name in zip(probs, im_names):\n names2probs[im_name] = prob\n TTA10.append(names2probs)\n for im_name in TTA10[0].keys():\n prob = (TTA10[0][im_name]+TTA10[1][im_name]+TTA10[2][im_name]+TTA10[3][im_name]+TTA10[4][im_name]\\\n +TTA10[5][im_name]+TTA10[6][im_name]+TTA10[7][im_name]+TTA10[8][im_name]+TTA10[9][im_name])/10\n top1_idx = prob.argsort()[-1]\n top1_name = list(cls2label.keys())[list(cls2label.values()).index(top1_idx)]\n results.append({'Id':im_name, 'Category':''.join(top1_name)})\n df = pd.DataFrame(results, columns=['Category', 'Id'])\n df.to_csv('sub_tta10.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lr Schedule\n* GradualWarmupScheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GradualWarmupScheduler(_LRScheduler):\n \"\"\" Gradually warm-up(increasing) learning rate in optimizer.\n Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.\n Args:\n optimizer (Optimizer): Wrapped optimizer.\n multiplier: target learning rate = base lr * multiplier\n total_epoch: target learning rate is reached at total_epoch, gradually\n after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau)\n \"\"\"\n\n def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n self.multiplier = multiplier\n if self.multiplier <= 1.:\n raise ValueError('multiplier should be greater than 1.')\n self.total_epoch = total_epoch\n self.after_scheduler = after_scheduler\n self.finished = False\n super().__init__(optimizer)\n\n def get_lr(self):\n if self.last_epoch > self.total_epoch:\n if self.after_scheduler:\n if not self.finished:\n self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n self.finished = True\n return self.after_scheduler.get_lr()\n return [base_lr * self.multiplier for base_lr in self.base_lrs]\n\n return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n\n def step_ReduceLROnPlateau(self, metrics, epoch=None):\n if epoch is None:\n epoch = self.last_epoch + 1\n self.last_epoch = epoch if epoch != 0 else 1 # ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning\n if self.last_epoch <= self.total_epoch:\n warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n for param_group, lr in zip(self.optimizer.param_groups, warmup_lr):\n param_group['lr'] = lr\n else:\n if epoch is None:\n self.after_scheduler.step(metrics, None)\n else:\n self.after_scheduler.step(metrics, epoch - self.total_epoch)\n\n def step(self, epoch=None, metrics=None):\n if type(self.after_scheduler) != ReduceLROnPlateau:\n if self.finished and self.after_scheduler:\n if epoch is None:\n self.after_scheduler.step(None)\n else:\n self.after_scheduler.step(epoch - self.total_epoch)\n else:\n return super(GradualWarmupScheduler, self).step(epoch)\n else:\n self.step_ReduceLROnPlateau(metrics, epoch)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Label smooth"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothSoftmaxCE(nn.Module):\n def __init__(self,\n lb_pos=0.9,\n lb_neg=0.005,\n reduction='mean',\n lb_ignore=255,\n ):\n super(LabelSmoothSoftmaxCE, self).__init__()\n self.lb_pos = lb_pos\n self.lb_neg = lb_neg\n self.reduction = reduction\n self.lb_ignore = lb_ignore\n self.log_softmax = nn.LogSoftmax(1)\n\n def forward(self, logits, label):\n logs = self.log_softmax(logits)\n ignore = label.data.cpu() == self.lb_ignore\n n_valid = (ignore == 0).sum()\n label[ignore] = 0\n lb_one_hot = logits.data.clone().zero_().scatter_(1, label.unsqueeze(1), 1)\n label = self.lb_pos * lb_one_hot + self.lb_neg * (1-lb_one_hot)\n ignore = ignore.nonzero()\n _, M = ignore.size()\n a, *b = ignore.chunk(M, dim=1)\n label[[a, torch.arange(label.size(1)), *b]] = 0\n\n if self.reduction == 'mean':\n loss = -torch.sum(torch.sum(logs*label, dim=1)) / n_valid\n elif self.reduction == 'none':\n loss = -torch.sum(logs*label, dim=1)\n return loss\n\n\nif __name__ == '__main__':\n torch.manual_seed(15)\n criteria = LabelSmoothSoftmaxCE(lb_pos=0.9, lb_neg=5e-3)\n \n inten = torch.randn(10, 5).cuda()\n lbs = torch.randint(5, (10,)).cuda()\n print('inten:', inten)\n print('lbs:', lbs)\n\n import torch.nn.functional as F\n\n loss = criteria(inten, lbs)\n print('loss:', loss)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixup"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n '''Returns mixed inputs, pairs of targets, and lambda'''\n if alpha > 0:\n lam = np.random.beta(alpha, alpha)\n else:\n lam = 1\n\n batch_size = x.size()[0]\n if use_cuda:\n index = torch.randperm(batch_size).cuda()\n else:\n index = torch.randperm(batch_size)\n\n mixed_x = lam * x + (1 - lam) * x[index, :]\n y_a, y_b = y, y[index]\n \n return mixed_x, y_a, y_b, lam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# main"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train():\n begin_time = time.time()\n # model\n if Model == 'resnet18':\n backbone = models.resnet18(pretrained=True)\n model = ResNet18(backbone, num_classes=Num_classes)\n metric_fc = ArcMarginProduct(512, Num_classes, s=30, m=0.5, easy_margin=False)\n elif Model == 'se_resnext50_32x4d':\n backbone = pretrainedmodels.se_resnext50_32x4d(pretrained='imagenet')\n model = se_resnext50_32x4d(backbone, 5)\n metric_fc = ArcMarginProduct(1024, Num_classes, s=30, m=0.5, easy_margin=False)\n # print(model)\n # model = torch.nn.DataParallel(model)\n model.cuda()\n metric_fc.cuda()\n \n # freeze layers\n if Freeze:\n if Model == 'se_resnext50_32x4d':\n for p in model.backbone.layer0.parameters(): p.requires_grad = False\n for p in model.backbone.layer1.parameters(): p.requires_grad = False\n for p in model.backbone.layer2.parameters(): p.requires_grad = False\n for p in model.backbone.layer3.parameters(): p.requires_grad = False\n # for p in model.backbone.layer4.parameters(): p.requires_grad = False\n\n # train data\n train_transform = transforms.Compose([transforms.Scale(256),\n transforms.RandomSizedCrop(224),\n transforms.RandomHorizontalFlip(),\n transforms.RandomVerticalFlip(),\n transforms.ColorJitter(0.05, 0.05, 0.05),\n transforms.RandomRotation(30),\n transforms.Resize((Size, Size)),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225])])\n dst_train = CDCDataset(ims2labels, transform=train_transform) # ims2labels_train\n dataloader_train = DataLoader(dst_train, shuffle=True, batch_size=Batch_size, num_workers=8)\n\n# # validation data\n# test_transform = transforms.Compose([transforms.Resize((Size, Size)),\n# transforms.ToTensor(),\n# transforms.Normalize(mean=[0.485, 0.456, 0.406], \n# std=[0.229, 0.224, 0.225])])\n# dst_valid = CDCDataset(ims2labels_val, transform=test_transform)\n# dataloader_valid = DataLoader(dst_valid, shuffle=False, batch_size=Batch_size//2, num_workers=8)\n\n # load checkpoint\n if Resume:\n model = torch.load(os.path.join('./checkpoints', Checkpoint))\n\n # train\n sum = 0\n train_loss_sum = 0\n train_top1_sum = 0\n \n # loss\n if Loss == 'CrossEntropy':\n criterion = nn.CrossEntropyLoss().cuda()\n elif Loss == 'FocalLoss':\n criterion = FocalLoss(Num_classes, alpha=Alpha, gamma=Gamma, size_average=True)\n elif Loss == 'LabelSmoothSoftmaxCE':\n criterion = LabelSmoothSoftmaxCE(lb_pos=0.9, lb_neg=0.05) # lb_neg=5e-3\n criterion.cuda()\n \n optimizer = torch.optim.Adam([{'params': filter(lambda p: p.requires_grad, model.parameters())}, {'params': metric_fc.parameters()}], \n lr=Init_lr, betas=(0.9, 0.999), weight_decay=0.0002)\n # lr schedule\n # scheduler_steplr = torch.optim.lr_scheduler.StepLR(optimizer, step_size=Step_size, gamma=0.1, last_epoch=-1)\n # warmup schedule\n scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, Num_epochs)\n scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=Multiplier, total_epoch=Total_epoch, after_scheduler=scheduler_cosine)\n \n \n print('Start training ...')\n train_loss_list, val_loss_list = [], []\n train_top1_list, val_top1_list = [], []\n lr_list = []\n for epoch in range(Num_epochs):\n # scheduler_steplr.step()\n scheduler_warmup.step()\n# print('Inference testset...')\n# inference(model)\n ep_start = time.time()\n # val_loss, val_top1 = eval(model, dataloader_valid, criterion)\n model.train()\n top1_sum = 0\n for i, (ims, labels) in enumerate(dataloader_train):\n input = Variable(ims).cuda()\n target = Variable(labels).cuda().long()\n\n feature = model(input)\n output = metric_fc(feature, target)\n \n loss = criterion(output, target)\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n top1 = accuracy(output.data, target.data, topk=(1,))\n train_loss_sum += loss.data.cpu().numpy()\n train_top1_sum += top1[0]\n sum += 1\n top1_sum += top1[0]\n \n lr = optimizer.state_dict()['param_groups'][0]['lr']\n# print('Epoch [%d/%d] | lr: %f | train@loss: %.4f train@top1: %.4f | val@loss:%.4f val@top1:%.4f | time:%.4f s'\n# %(epoch+1, Num_epochs, lr, train_loss_sum/sum, train_top1_sum/sum, val_loss, val_top1, time.time()-ep_start))\n print('Epoch [%d/%d] | lr: %f | train@loss: %.4f train@top1: %.4f | time:%.4f s'\n %(epoch+1, Num_epochs, lr, train_loss_sum/sum, train_top1_sum/sum, time.time()-ep_start))\n train_loss_list.append(train_loss_sum/sum)\n # val_loss_list.append(val_loss)\n train_top1_list.append(train_top1_sum/sum)\n # val_top1_list.append(val_top1)\n lr_list.append(lr)\n \n sum = 0\n train_loss_sum = 0\n train_top1_sum = 0\n \n if (epoch+1) % 50 == 0 and epoch < Num_epochs or (epoch+1) == Num_epochs:\n print('Taking snapshot...')\n torch.save(model, '{}.pth'.format(Checkpoint))\n \n if (time.time()-begin_time)/60/60 > 8:\n break\n \n inference(model)\n inference_TTA(model)\n # draw curve\n figs = plt.figure()\n fig1 = figs.add_subplot(3, 1, 1)\n fig2 = figs.add_subplot(3, 1, 2)\n fig3 = figs.add_subplot(3, 1, 3)\n x = [i for i in range(len(train_loss_list))]\n fig1.plot(x, train_loss_list, label='train loss')\n # fig1.plot(x, val_loss_list, label='valid loss')\n fig1.legend(loc='upper right')\n\n fig2.plot(x, train_top1_list, label='train loss')\n # fig2.plot(x, val_top1_list, label='valid loss')\n fig2.legend(loc='bottom right')\n\n fig3.plot(x, lr_list, label='lr')\n fig3.legend(loc='upper right')\n\n # plt.xlabel('epoch')\n # plt.ylabel('loss')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n setup_seed(88)\n train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}