{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [ViT For Dog Breed Identification](https://www.kaggle.com/superbigfive/vit-for-dog-breed-identification)\n- 作者 : 常铭，电子科技大学，计算机科学与工程学院 (网络空间安全学院)；\n- 时间 : 2023-10-26;\n- 主要内容：基于 ViT 的细粒度狗品种图像分类；\n- 本实验在 Kaggle 网站上运行，深度学习框架为 Pytorch，从标题链接中 fork 一份即可一键运行；\n- 数据集选取 Kaggle 某场比赛中提供的[数据集](https://www.kaggle.com/competitions/dog-breed-identification)，大概一万张图片，共 120 种狗。实际上是 ImageNet 的一个子集；\n- 网络模型使用的 ViT，利用 torchvision 库的 models 方法快速构建，未启用预训练；\n- 损失函数设置为交叉熵损失函数，优化器选取 Adam，学习率、weight_decay 分别取 0.01, 0.0001；利用了 CosineAnnealingLR 学习率调度器实现训练过程中学习率的动态调整。","metadata":{"papermill":{"duration":0.008867,"end_time":"2023-10-24T12:24:42.523109","exception":false,"start_time":"2023-10-24T12:24:42.514242","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 📚 Import Libraries\n导入本次实验所需要的库。 <br/>","metadata":{"papermill":{"duration":0.007951,"end_time":"2023-10-24T12:24:42.539364","exception":false,"start_time":"2023-10-24T12:24:42.531413","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2","metadata":{"papermill":{"duration":0.03925,"end_time":"2023-10-24T12:24:42.586744","exception":false,"start_time":"2023-10-24T12:24:42.547494","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:18.742314Z","iopub.execute_input":"2023-10-27T03:06:18.743135Z","iopub.status.idle":"2023-10-27T03:06:18.774178Z","shell.execute_reply.started":"2023-10-27T03:06:18.743102Z","shell.execute_reply":"2023-10-27T03:06:18.773473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torchvision import models\nfrom torch.cuda import amp\n\n# 绘图相关\nfrom matplotlib import pyplot as plt\nimport matplotlib_inline\nfrom pylab import mpl\n# 指定默认字体：解决plot不能显示中文问题\nmpl.rcParams['font.sans-serif'] = ['Microsoft YaHei'] \n# 解决保存图像是负号'-'显示为方块的问题\nmpl.rcParams['axes.unicode_minus'] = False \n\nimport os\nimport numpy as np\nimport copy, random, time\n\nimport gc\n\n# 数据集相关\nimport cv2\nimport pandas as pd\nfrom PIL import Image\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import train_test_split\n\nfrom IPython import display as ipd\nfrom tqdm.notebook import tqdm\n# 使得终端可以打印不同颜色的文字\nfrom colorama import Fore, Back, Style\nc_  = Fore.GREEN\nsr_ = Style.RESET_ALL\nfrom collections import defaultdict\n\nimport warnings\nwarnings.filterwarnings (\"ignore\")","metadata":{"papermill":{"duration":27.484432,"end_time":"2023-10-24T12:25:10.079645","exception":false,"start_time":"2023-10-24T12:24:42.595213","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:18.776080Z","iopub.execute_input":"2023-10-27T03:06:18.776684Z","iopub.status.idle":"2023-10-27T03:06:24.570831Z","shell.execute_reply.started":"2023-10-27T03:06:18.776650Z","shell.execute_reply":"2023-10-27T03:06:24.569927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⭐ WandB\nWeights & Bias (W&B) 是一个可以跟踪实验的平台，利用该平台可以很方便地可视化实验过程中各项指标，进行对比分析，便于我们调整模型的各项参数。","metadata":{"papermill":{"duration":0.00869,"end_time":"2023-10-24T12:25:10.097233","exception":false,"start_time":"2023-10-24T12:25:10.088543","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -qq -U wandb\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient ()\napi_key = user_secrets.get_secret (\"WANDB\")\nwandb.login (key = api_key)\nanonymous = None","metadata":{"papermill":{"duration":1.981828,"end_time":"2023-10-24T12:25:12.088237","exception":false,"start_time":"2023-10-24T12:25:10.106409","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:24.571896Z","iopub.execute_input":"2023-10-27T03:06:24.572288Z","iopub.status.idle":"2023-10-27T03:06:44.896343Z","shell.execute_reply.started":"2023-10-27T03:06:24.572263Z","shell.execute_reply":"2023-10-27T03:06:44.894998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚙️ Configuration\n本实验中所有用到的参数都将统一设置在 CFG 类中，便于查看与修改。","metadata":{"papermill":{"duration":0.008689,"end_time":"2023-10-24T12:25:12.105868","exception":false,"start_time":"2023-10-24T12:25:12.097179","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG :\n    seed          = 20231026\n    exp_name      = 'ViT For Dog Breed Identification'\n    model_name    = 'vit_l_16'\n    train_bs      = 32\n    valid_bs      = 32\n    test_bs       = 32\n    num_workers   = 2\n    pre_size      = [243, 324]\n    img_size      = [224, 224]\n    epochs        = 25\n    lr            = 1e-2\n    momentum      = 0.9\n    wd            = 1e-4\n    scheduler     = 'CosineAnnealingLR'\n    optimizer     = 'Adam'\n    min_lr        = 1e-6\n    T_max         = int (30000 / train_bs * epochs) + 50\n    T_0           = 25\n    warmup_epochs = 0\n    n_accumulate  = max (1, 32 // train_bs)\n    num_classes   = 120\n    device        = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"papermill":{"duration":0.13232,"end_time":"2023-10-24T12:25:12.247131","exception":false,"start_time":"2023-10-24T12:25:12.114811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:44.899321Z","iopub.execute_input":"2023-10-27T03:06:44.900343Z","iopub.status.idle":"2023-10-27T03:06:45.002648Z","shell.execute_reply.started":"2023-10-27T03:06:44.900299Z","shell.execute_reply":"2023-10-27T03:06:45.001487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ❗ Reproducibility\n这里设置了所有随机种子的值，保证了每次运行时，运行结果都是相同的。","metadata":{"papermill":{"duration":0.009123,"end_time":"2023-10-24T12:25:12.266741","exception":false,"start_time":"2023-10-24T12:25:12.257618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def set_seed (seed = CFG.seed):\n    '''\n        初始化各个随机种子为同一值, \n        保证每次执行程序的运行情况都相同\n    '''\n    np.random.seed (seed)\n    random.seed (seed)\n    torch.manual_seed (seed)\n    torch.cuda.manual_seed (seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str (seed)\n    print ('> SEEDING DONE')\n    \nset_seed (CFG.seed)","metadata":{"papermill":{"duration":0.077811,"end_time":"2023-10-24T12:25:12.353528","exception":false,"start_time":"2023-10-24T12:25:12.275717","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:45.004009Z","iopub.execute_input":"2023-10-27T03:06:45.004386Z","iopub.status.idle":"2023-10-27T03:06:45.070336Z","shell.execute_reply.started":"2023-10-27T03:06:45.004354Z","shell.execute_reply":"2023-10-27T03:06:45.069293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔨 Utility\n基于本实验设置的一些简单的工具。","metadata":{"papermill":{"duration":0.008594,"end_time":"2023-10-24T12:25:12.370948","exception":false,"start_time":"2023-10-24T12:25:12.362354","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def load_img (path):\n    '''\n        根据文件路径读取图像, 并缩放像素值大小至 [0, 1]\n    '''\n    img = cv2.imread (path, cv2.IMREAD_UNCHANGED)\n    img = img.astype ('float32') # original is uint16\n#     mx = np.max (img)\n#     if mx : img /= mx\n    img /= 255.0\n    return img\n\ndef showImage (imgs, labels, size = 5) :\n    '''\n        输入是若干张图片组成的一个 list\n        每张图片的格式都是张量，数据范围是 [0, 1]\n    '''\n    size = min (size, len (imgs))\n    plt.figure (figsize = (5 * size, 5))\n    for i in range (size) :\n        plt.subplot (1, size, i + 1)\n        img = imgs[i, ].detach ().cpu ()\n        img = img.view (3, -1) * channel_std.view(3, -1) + channel_mean.view (3, -1)\n        img = img.view (3, CFG.img_size[0], CFG.img_size[1])\n        img = img.permute (1, 2, 0)\n        plt.axis ('off')\n        plt.title (num2label[int (labels[i])], fontsize = 'xx-large')\n        plt.imshow (img)\n    plt.tight_layout ()\n    plt.show ()\n    ","metadata":{"papermill":{"duration":0.07304,"end_time":"2023-10-24T12:25:12.453117","exception":false,"start_time":"2023-10-24T12:25:12.380077","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:45.071581Z","iopub.execute_input":"2023-10-27T03:06:45.071865Z","iopub.status.idle":"2023-10-27T03:06:45.133574Z","shell.execute_reply.started":"2023-10-27T03:06:45.071841Z","shell.execute_reply":"2023-10-27T03:06:45.132549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍚 Dataset\n创建一个继承 $Dataset$ 类的子类, 重写 $\\_ \\_ len \\_ \\_$ 和 $\\_ \\_ getitem \\_ \\_$ 方法, 前者用来获取数据及大小, 后者用于指定下标, 返回对应的样本。","metadata":{"papermill":{"duration":0.008676,"end_time":"2023-10-24T12:25:12.470889","exception":false,"start_time":"2023-10-24T12:25:12.462213","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class BuildDataset (torch.utils.data.Dataset):\n    def __init__(self, df, Datatype, transforms = None) :\n        self.transforms = transforms\n        self.Datatype   = Datatype\n\n        self.img_paths  = [] \n        self.labels     = []\n        \n        if self.Datatype is not \"test\" :\n            BASE_PATH  = \"../input/dog-breed-identification/train\"\n            for img_name, label in zip (df['id'].values, df['breed']) :\n                self.img_paths.append (f\"{BASE_PATH}/{img_name}.jpg\")\n                self.labels.append (label)\n        else :\n            BASE_PATH = \"../input/dog-breed-identification/test\"\n            for img_name in os.listdir (BASE_PATH) :\n                self.img_paths.append (f\"{BASE_PATH}/{img_name}\")  \n        \n    def __len__ (self) :\n        return len (self.img_paths)\n    \n    def __getitem__ (self, idx) :\n        img_path = self.img_paths[idx]\n        img = Image.open (img_path).convert (\"RGB\")\n        img = np.asarray (img)\n        if self.transforms :\n            data = self.transforms[self.Datatype] (image = img)\n            img  = data['image']\n        img = np.transpose (img, (2, 0, 1))\n        if self.Datatype is not \"test\" :\n            label = self.labels[idx]\n            return torch.tensor (img), torch.tensor (label2num[label])\n        else :\n            return torch.tensor (img)","metadata":{"papermill":{"duration":0.077506,"end_time":"2023-10-24T12:25:12.557062","exception":false,"start_time":"2023-10-24T12:25:12.479556","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:45.134766Z","iopub.execute_input":"2023-10-27T03:06:45.135108Z","iopub.status.idle":"2023-10-27T03:06:45.197038Z","shell.execute_reply.started":"2023-10-27T03:06:45.135070Z","shell.execute_reply":"2023-10-27T03:06:45.196095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🌈 Augmentations\n对训练集图像进行数据增强，包含以下操作：<br/>\n- 对图像进行中心裁剪操作；\n- 对图像进行水平翻转操作；\n- 对图像进行随机平移、缩放、旋转操作；\n- 对图像进行网格畸变或弹性变换操作；\n\n由于原图尺寸较大，故对原图像进行了中心裁剪，减少了模型训练计算量；<br/>\n另外，因为该分类任务是细粒度的分类任务，往往一些局部信息更能决定图像的类别。中心裁剪操作事实上也可以看作是局部放大操作，并且观察训练集中部分图像后，发现存在一些狗在图像中占比较小的情况。所以，中心裁剪操作对模型精度的提升也起到了一定的作用。其余的一些图像增强操作主要是减少模型的过拟合情况。<br/>\n模型启用了预训练，预训练时采用的数据集是 ImageNet，且对图像进行了归一化操作，所以为了使启用预训练的模型效果更好，对图像进行了与在 ImageNet 数据集上相同的归一化操作。","metadata":{"papermill":{"duration":0.008699,"end_time":"2023-10-24T12:25:12.574922","exception":false,"start_time":"2023-10-24T12:25:12.566223","status":"completed"},"tags":[]}},{"cell_type":"code","source":"channel_mean = torch.Tensor ([0.485, 0.456, 0.406])\nchannel_std = torch.Tensor ([0.229, 0.224, 0.225])\n\ndata_transforms = {\n    \"train\" : A.Compose ([\n        # 固定图像尺寸\n        A.Resize (*CFG.pre_size),\n        A.CenterCrop (*CFG.img_size),\n        # 随机水平翻转\n        A.HorizontalFlip (p = 0.6), \n        # 随机平移、缩放、旋转\n        A.ShiftScaleRotate (shift_limit = 0.0625, scale_limit = 0.05, rotate_limit = 10, p = 0.5), \n        A.OneOf ([\n            # 网格畸变\n            A.GridDistortion (num_steps = 5, distort_limit = 0.05, p = 1.0), \n            # 弹性变换, 扭曲图像的同时保持图像的连续性\n            A.ElasticTransform (alpha = 1, sigma = 10, alpha_affine = 10, p = 1.0)\n        ], p = 0.25),\n        A.Normalize (channel_mean, channel_std),\n        ], p = 1.0),\n    \n    \"valid\" : A.Compose ([\n        A.Resize (*CFG.pre_size, interpolation = cv2.INTER_NEAREST),\n        A.CenterCrop (*CFG.img_size),\n        A.Normalize (channel_mean, channel_std)\n        ], p = 1.0),\n    \n    \"test\" : A.Compose ([\n        A.Resize (*CFG.pre_size, interpolation = cv2.INTER_NEAREST),\n        A.CenterCrop (*CFG.img_size),\n        A.Normalize (channel_mean, channel_std)\n        ], p = 1.0)\n}","metadata":{"papermill":{"duration":0.067061,"end_time":"2023-10-24T12:25:12.650593","exception":false,"start_time":"2023-10-24T12:25:12.583532","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:45.198416Z","iopub.execute_input":"2023-10-27T03:06:45.198696Z","iopub.status.idle":"2023-10-27T03:06:45.274261Z","shell.execute_reply.started":"2023-10-27T03:06:45.198673Z","shell.execute_reply":"2023-10-27T03:06:45.273395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍰 DataLoader\n导入数据集。","metadata":{"papermill":{"duration":0.00854,"end_time":"2023-10-24T12:25:12.667906","exception":false,"start_time":"2023-10-24T12:25:12.659366","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def DataLoader (df, data_transforms) :\n    train_and_test_dataset = BuildDataset (df, \"train\", transforms = data_transforms)\n    indexes = list (range (len (train_and_test_dataset)))\n    train_indexes, valid_indexes = train_test_split(indexes, test_size = 0.1)\n    train_dataset = torch.utils.data.Subset (train_and_test_dataset, train_indexes)\n    valid_dataset = torch.utils.data.Subset (train_and_test_dataset, valid_indexes)\n    test_dataset  = BuildDataset (df, \"test\", transforms = data_transforms)\n    valid_dataset.Datatype = \"valid\"\n    \n    # pin_memory 参数表示是否将加载的数据常驻内存\n    # drop_last 参数表示是否丢弃最后一个批次 (可能不满 batch_size 个样本)\n    train_loader = torch.utils.data.DataLoader (train_dataset, batch_size = CFG.train_bs, num_workers = CFG.num_workers, \n                                                shuffle = True, pin_memory = True, drop_last = False)\n    valid_loader = torch.utils.data.DataLoader (valid_dataset, batch_size = CFG.valid_bs, num_workers = CFG.num_workers, \n                                                shuffle = False, pin_memory = True, drop_last = False)\n    test_loader  = torch.utils.data.DataLoader (test_dataset, batch_size = CFG.test_bs, num_workers = CFG.num_workers,\n                                                shuffle = False, pin_memory = True, drop_last = False)\n    \n    return train_loader, valid_loader, test_loader\n\ncsv_path = \"/kaggle/input/dog-breed-identification/labels.csv\"\ndf = pd.read_csv (csv_path)\nclasses = list (set (df['breed'].values))\nlabel2num = dict (zip (classes, range (len (classes))))\nnum2label = dict (zip (range (len (classes)), classes))\n\ntrain_loader, valid_loader, test_loader = DataLoader (df, data_transforms)\n\nprint (\"len of batch : \", len (train_loader))\nit = iter (train_loader)\nimgs, labels = next (it)\nprint (imgs.size (), labels.size ())","metadata":{"papermill":{"duration":5.679074,"end_time":"2023-10-24T12:25:18.355706","exception":false,"start_time":"2023-10-24T12:25:12.676632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:45.275550Z","iopub.execute_input":"2023-10-27T03:06:45.275885Z","iopub.status.idle":"2023-10-27T03:06:48.881351Z","shell.execute_reply.started":"2023-10-27T03:06:45.275853Z","shell.execute_reply":"2023-10-27T03:06:48.879346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📈 Visualization\n随机展示部分图像，检查数据集的读入是否出错、检测数据增强效果。","metadata":{"papermill":{"duration":0.00889,"end_time":"2023-10-24T12:25:18.373520","exception":false,"start_time":"2023-10-24T12:25:18.364630","status":"completed"},"tags":[]}},{"cell_type":"code","source":"showImage (imgs, labels)","metadata":{"papermill":{"duration":1.622216,"end_time":"2023-10-24T12:25:20.004529","exception":false,"start_time":"2023-10-24T12:25:18.382313","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:48.886974Z","iopub.execute_input":"2023-10-27T03:06:48.887843Z","iopub.status.idle":"2023-10-27T03:06:50.527284Z","shell.execute_reply.started":"2023-10-27T03:06:48.887803Z","shell.execute_reply":"2023-10-27T03:06:50.525731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📦 Model\n模型的构建部分，利用快速构建 Unet 模型，并选取 efficientnet-b0 作为 Unet 的编码器部分。","metadata":{"papermill":{"duration":0.03175,"end_time":"2023-10-24T12:25:20.068598","exception":false,"start_time":"2023-10-24T12:25:20.036848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class ViT (nn.Module) :\n    def __init__ (self) :\n        super (ViT, self).__init__ ()\n        model = models.vit_l_16 (pretrained = True)\n        input_size = model.heads.head.in_features\n        output_size = CFG.num_classes\n#         model.heads.head = nn.Sequential (\n#             nn.Linear (input_size, 512),\n#             nn.ReLU (),\n#             nn.Linear (512, output_size)\n#         )\n        model.heads.head = nn.Linear (input_size, output_size)\n        self.model = model\n        \n        # 冻结参数，意味着只训练分类头。由于未启用预训练，故不做此操作\n        for param in self.model.named_parameters () :\n            if \"heads\" not in param[0]:\n                param[1].requires_grad = False\n\n    def forward (self, x):\n        return self.model (x)\n\ndef LoadModel (LOADPATH) :\n    '''\n        用于测试模型前加载训练过程中表现最好的模型\n    '''\n    model = ViT ()\n    model.load_state_dict (torch.load (LOADPATH))\n    model.eval ()\n    return model","metadata":{"papermill":{"duration":0.093753,"end_time":"2023-10-24T12:25:20.195008","exception":false,"start_time":"2023-10-24T12:25:20.101255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.528804Z","iopub.execute_input":"2023-10-27T03:06:50.529209Z","iopub.status.idle":"2023-10-27T03:06:50.596061Z","shell.execute_reply.started":"2023-10-27T03:06:50.529179Z","shell.execute_reply":"2023-10-27T03:06:50.595025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔧 Loss Function\n损失函数使用交叉熵损失函数，同时实现了计算准确个数的函数。","metadata":{"papermill":{"duration":0.031487,"end_time":"2023-10-24T12:25:20.258561","exception":false,"start_time":"2023-10-24T12:25:20.227074","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def calc_correct (preds, labels) :\n    _, preds = torch.max (preds, 1)\n    return (preds == labels).float ().sum ().cpu ().item ()\n\ndef criterion (preds, labels) :\n    return nn.CrossEntropyLoss () (preds, labels).sum ()","metadata":{"papermill":{"duration":0.097262,"end_time":"2023-10-24T12:25:20.387213","exception":false,"start_time":"2023-10-24T12:25:20.289951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.597301Z","iopub.execute_input":"2023-10-27T03:06:50.597600Z","iopub.status.idle":"2023-10-27T03:06:50.655896Z","shell.execute_reply.started":"2023-10-27T03:06:50.597575Z","shell.execute_reply":"2023-10-27T03:06:50.654920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔍 Optimizer\ntorch.optim.CosineAnnealingLR 方法是一个用于设置学习率的调度器，它可以根据一个余弦退火的策略，动态地调整优化器的学习率。它的原理是在每个周期内，将学习率从一个最大值降低到一个最小值，然后在下一个周期内重复这个过程。这样可以避免学习率过大或过小导致的收敛困难或局部最优;","metadata":{"papermill":{"duration":0.033605,"end_time":"2023-10-24T12:25:20.456500","exception":false,"start_time":"2023-10-24T12:25:20.422895","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def fetch_scheduler (optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR' :\n        scheduler = lr_scheduler.CosineAnnealingLR (optimizer, T_max = CFG.T_max, \n                                                   eta_min = CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts' :\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts (optimizer, T_0 = CFG.T_0, \n                                                             eta_min = CFG.min_lr)\n    elif CFG.scheduler == 'ReduceLROnPlateau' :\n        scheduler = lr_scheduler.ReduceLROnPlateau (optimizer,\n                                                   mode = 'min',\n                                                   factor = 0.1,\n                                                   patience = 7,\n                                                   threshold = 0.0001,\n                                                   min_lr = CFG.min_lr,)\n    elif CFG.scheduer == 'ExponentialLR' :\n        scheduler = lr_scheduler.ExponentialLR (optimizer, gamma = 0.85)\n    elif CFG.scheduler == None:\n        return None\n        \n    return scheduler","metadata":{"papermill":{"duration":0.092619,"end_time":"2023-10-24T12:25:20.581087","exception":false,"start_time":"2023-10-24T12:25:20.488468","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.657050Z","iopub.execute_input":"2023-10-27T03:06:50.657447Z","iopub.status.idle":"2023-10-27T03:06:50.719282Z","shell.execute_reply.started":"2023-10-27T03:06:50.657422Z","shell.execute_reply":"2023-10-27T03:06:50.718272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚄 Training Function\n每一个 epoch 训练过程的具体实现。模型训练采用了自动混合精度策略。<br/>","metadata":{"papermill":{"duration":0.03181,"end_time":"2023-10-24T12:25:20.645312","exception":false,"start_time":"2023-10-24T12:25:20.613502","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_one_epoch (model, optimizer, scheduler, dataloader, device, epoch) :\n    model.train ()\n    # 创建一个 GradScaler 对象, 可以在迭代过程中动态估计损失放大的倍数\n    scaler = amp.GradScaler ()\n    \n    dataset_size = 0\n    running_loss, num_correct = 0.0, 0\n    epoch_loss, epoch_accuracy = 0.0, 0.0\n    \n    # 将一个可迭代对象作为参数传入，然后返回一个包装后的可迭代对象，\n    # 可以像平常一样对其进行迭代，每次请求一个值时，都会打印一个进度条。\n    pbar = tqdm (enumerate (dataloader), total = len (dataloader), desc = 'Train ')\n    for step, (images, labels) in pbar:         \n        images = images.to (device, dtype = torch.float)\n        labels = labels.to (device)\n        batch_size = images.size (0)\n        \n        # 前向传播过程中自动混合精度训练\n        with amp.autocast (enabled = True):\n            y_pred    = model (images)\n            y_pred    = y_pred.to (device, dtype = torch.float)\n            loss      = criterion (y_pred, labels)\n            \n        running_loss += loss.item ()\n        num_correct  += calc_correct (y_pred, labels)\n        # 放大损失、反向传播\n        scaler.scale (loss).backward ()\n        if (step + 1) % CFG.n_accumulate == 0 :\n            # 根据原放大倍数，梯度更新时缩小相应的倍数\n            scaler.step (optimizer)\n            # 更新损失放大的倍数\n            scaler.update ()\n            optimizer.zero_grad ()\n            if scheduler is not None :\n                # 更新学习率\n                scheduler.step ()\n        \n        dataset_size += batch_size\n        \n        epoch_loss     = running_loss / dataset_size\n        epoch_accuracy = num_correct / dataset_size\n        \n        mem = torch.cuda.memory_reserved () / 1E9 if torch.cuda.is_available () else 0\n        current_lr = optimizer.param_groups[0]['lr']\n        pbar.set_postfix (train_loss = f'{epoch_loss : 0.4f}',\n                          train_accuracy = f'{epoch_accuracy : 0.4f}',\n                          lr = f'{current_lr : 0.5f}',\n                          gpu_mem = f'{mem : 0.2f} GB')\n    torch.cuda.empty_cache ()\n    gc.collect ()\n    \n    return epoch_loss","metadata":{"papermill":{"duration":0.093927,"end_time":"2023-10-24T12:25:20.771215","exception":false,"start_time":"2023-10-24T12:25:20.677288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.720991Z","iopub.execute_input":"2023-10-27T03:06:50.721756Z","iopub.status.idle":"2023-10-27T03:06:50.788361Z","shell.execute_reply.started":"2023-10-27T03:06:50.721722Z","shell.execute_reply":"2023-10-27T03:06:50.787280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 👀 Validation Function\n评估函数的实现，使用验证集评估模型的分类效果和泛化性能。","metadata":{}},{"cell_type":"code","source":"@torch.no_grad ()\ndef valid_one_epoch (model, dataloader, device, epoch):\n    model.eval ()\n    \n    dataset_size = 0\n    running_loss, num_correct = 0.0, 0\n    epoch_loss, epoch_accuracy = 0.0, 0.0\n    \n    pbar = tqdm (enumerate (dataloader), total = len (dataloader), desc = 'Valid ')\n    for step, (images, labels) in pbar :\n        images  = images.to (device, dtype = torch.float)\n        labels  = labels.to (device)\n        batch_size = images.size (0)\n        \n        y_pred    = model (images)\n        y_pred    = y_pred.to (device, dtype = torch.float)\n        loss      = criterion (y_pred, labels)\n        \n        running_loss += loss.item ()\n        num_correct  += calc_correct (y_pred, labels)\n        dataset_size += batch_size\n        \n        epoch_loss     = running_loss / dataset_size\n        epoch_accuracy = num_correct / dataset_size\n        \n        mem = torch.cuda.memory_reserved () / 1E9 if torch.cuda.is_available() else 0\n        current_lr = optimizer.param_groups[0]['lr']\n        pbar.set_postfix (valid_loss = f'{epoch_loss : 0.4f}',\n                          valid_accuracy = f'{epoch_accuracy : 0.4f}',\n                          lr = f'{current_lr : 0.5f}',\n                          gpu_mem = f'{mem : 0.2f} GB')\n    torch.cuda.empty_cache ()\n    gc.collect()\n    \n    return epoch_loss, epoch_accuracy","metadata":{"execution":{"iopub.status.busy":"2023-10-27T03:06:50.789696Z","iopub.execute_input":"2023-10-27T03:06:50.790091Z","iopub.status.idle":"2023-10-27T03:06:50.854877Z","shell.execute_reply.started":"2023-10-27T03:06:50.790058Z","shell.execute_reply":"2023-10-27T03:06:50.853795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🏃 Run Training\n每一轮训练过后，需要做一些实验数据相关的统计 (通过 wandb 来实现)。","metadata":{"papermill":{"duration":0.031518,"end_time":"2023-10-24T12:25:20.834873","exception":false,"start_time":"2023-10-24T12:25:20.803355","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def run_training (model, run, optimizer, scheduler, device, num_epochs) :\n    # wandb 自动记录 PyTorch 模型的权重、偏置和梯度, log_freq 设置记录的频率 (每 log_freq 批次记录一次)\n    wandb.watch (model, log_freq = 100)\n    \n    # 打印 GPU 的名字\n    if torch.cuda.is_available () :\n        print (\"cuda: {}\\n\".format (torch.cuda.get_device_name ()))\n    \n    start = time.time ()\n    best_model_wts = copy.deepcopy (model.state_dict ())\n    best_accuracy  = -np.inf\n    best_epoch     = -1\n    history        = defaultdict (list)\n    \n    for epoch in range (1, num_epochs + 1): \n        gc.collect ()\n        print (f'Epoch {epoch} / {num_epochs}', end = '')\n        train_loss = train_one_epoch (model, optimizer, scheduler, \n                                      dataloader = train_loader, \n                                      device = CFG.device, epoch = epoch)\n  \n\n        val_loss, val_accuracy = valid_one_epoch (model, valid_loader,\n                                                  device = CFG.device, \n                                                  epoch = epoch)\n        history['Train Loss'].append (train_loss)\n        history['Valid Loss'].append (val_loss)\n        history['Valid Accuracy'].append (val_accuracy)\n        \n        # 记录、更新模型的性能指标\n        wandb.log ({\"Train Loss\" : train_loss, \n                   \"Valid Loss\" : val_loss,\n                   \"Valid Accuracy\" : val_accuracy,\n                   \"LR\" : scheduler.get_last_lr ()[0]})\n        \n        # 保存更优的模型\n        if val_accuracy >= best_accuracy :\n            print(f\"{c_}Valid Accuracy Improved ({best_accuracy:0.4f} ---> {val_accuracy:0.4f})\")\n            best_accuracy = val_accuracy\n            best_epoch    = epoch\n            run.summary[\"Best Accuracy\"] = best_accuracy\n            run.summary[\"Best Epoch\"]    = best_epoch\n            best_model_wts = copy.deepcopy (model.state_dict ())\n            PATH = f\"best_model.bin\"\n            torch.save (model.state_dict (), PATH)\n            # Save a model file from the current directory\n            wandb.save (PATH)\n            print (f\"Model Saved{sr_}\")\n            \n        last_model_wts = copy.deepcopy (model.state_dict ())\n        PATH = f\"last_model.bin\"\n        torch.save (model.state_dict (), PATH)\n            \n        print (); print ()\n    \n    end = time.time ()\n    time_elapsed = end - start\n    print ('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format (\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print (\"Best Accuracy: {:.4f}\".format (best_accuracy))\n    \n    # load best model weights\n    model.load_state_dict (best_model_wts)\n    \n    return model, history","metadata":{"papermill":{"duration":0.097935,"end_time":"2023-10-24T12:25:20.964422","exception":false,"start_time":"2023-10-24T12:25:20.866487","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.856317Z","iopub.execute_input":"2023-10-27T03:06:50.856635Z","iopub.status.idle":"2023-10-27T03:06:50.922452Z","shell.execute_reply.started":"2023-10-27T03:06:50.856598Z","shell.execute_reply":"2023-10-27T03:06:50.921425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚅 Training\n真的开始训练了。","metadata":{"papermill":{"duration":0.085885,"end_time":"2023-10-24T12:25:21.082553","exception":false,"start_time":"2023-10-24T12:25:20.996668","status":"completed"},"tags":[]}},{"cell_type":"code","source":"run = wandb.init (project = CFG.exp_name, \n                 config = {k : v for k, v in dict (vars (CFG)).items () if '__' not in k},\n                 anonymous = anonymous,\n                 name = f\"model-{CFG.model_name}|dim-{CFG.img_size[0]}x{CFG.img_size[1]}|optimizer-{CFG.optimizer}\",\n                )\ntrain_loader, valid_loader, test_loader = DataLoader (df, data_transforms)\nmodel     = ViT ().to (CFG.device)\noptimizer = torch.optim.Adam (model.parameters (), lr = CFG.lr, weight_decay = CFG.wd)\nscheduler = fetch_scheduler (optimizer)\nmodel, history = run_training (model, run, optimizer, scheduler,\n                              device = CFG.device,\n                              num_epochs = CFG.epochs)\nrun.finish ()\ndisplay (ipd.IFrame (run.url, width = 1000, height = 720))","metadata":{"papermill":{"duration":2182.153172,"end_time":"2023-10-24T13:01:43.268138","exception":false,"start_time":"2023-10-24T12:25:21.114966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T03:06:50.923839Z","iopub.execute_input":"2023-10-27T03:06:50.924195Z","iopub.status.idle":"2023-10-27T04:32:06.595369Z","shell.execute_reply.started":"2023-10-27T03:06:50.924163Z","shell.execute_reply":"2023-10-27T04:32:06.594476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔭 Prediction\n","metadata":{"papermill":{"duration":0.161333,"end_time":"2023-10-24T13:01:43.590522","exception":false,"start_time":"2023-10-24T13:01:43.429189","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def Precision (model, device) :\n    _, valid_loader, test_loader = DataLoader (df, data_transforms)\n    it = iter (test_loader)\n    for i in range (3) :\n        imgs = next (it)\n        imgs = imgs.to (device, dtype = torch.float)\n        preds = model (imgs)\n        _, preds = torch.max (preds, dim = 1)\n        showImage (imgs, preds, size = 5)\n        \nPrecision (model, device = CFG.device)","metadata":{"papermill":{"duration":11.134972,"end_time":"2023-10-24T13:01:54.886337","exception":false,"start_time":"2023-10-24T13:01:43.751365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-27T04:32:06.596703Z","iopub.execute_input":"2023-10-27T04:32:06.597003Z","iopub.status.idle":"2023-10-27T04:32:13.011640Z","shell.execute_reply.started":"2023-10-27T04:32:06.596977Z","shell.execute_reply":"2023-10-27T04:32:13.010456Z"},"trusted":true},"execution_count":null,"outputs":[]}]}