{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 第七章 现代卷积神经网络","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## 7.1. 深度卷积神经网络（AlexNet）","metadata":{}},{"cell_type":"markdown","source":"### 7.1.1. 学习表征","metadata":{}},{"cell_type":"markdown","source":"#### 7.1.1.1. 缺少的成分：数据","metadata":{}},{"cell_type":"markdown","source":"#### 7.1.1.2. 缺少的成分：硬件","metadata":{}},{"cell_type":"markdown","source":"### 7.1.2. AlexNet","metadata":{}},{"cell_type":"markdown","source":"#### 7.1.2.1. 模型设计","metadata":{}},{"cell_type":"markdown","source":"#### 7.1.2.2. 激活函数","metadata":{}},{"cell_type":"markdown","source":"#### 7.1.2.3. 容量控制和预处理","metadata":{}},{"cell_type":"code","source":"!pip install d2l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:11:57.596329Z","iopub.execute_input":"2025-06-11T09:11:57.596530Z","iopub.status.idle":"2025-06-11T09:12:17.620268Z","shell.execute_reply.started":"2025-06-11T09:11:57.596506Z","shell.execute_reply":"2025-06-11T09:12:17.619330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom d2l import torch as d2l\n\nnet = nn.Sequential(\n    # 这里使用一个11*11的更大窗口来捕捉对象。\n    # 同时，步幅为4，以减少输出的高度和宽度。\n    # 另外，输出通道的数目远大于LeNet\n    nn.Conv2d(1, 96, kernel_size=11, stride=4, padding=1), nn.ReLU(),\n    nn.MaxPool2d(kernel_size=3, stride=2),\n    # 减小卷积窗口，使用填充为2来使得输入与输出的高和宽一致，且增大输出通道数\n    nn.Conv2d(96, 256, kernel_size=5, padding=2), nn.ReLU(),\n    nn.MaxPool2d(kernel_size=3, stride=2),\n    # 使用三个连续的卷积层和较小的卷积窗口。\n    # 除了最后的卷积层，输出通道的数量进一步增加。\n    # 在前两个卷积层之后，汇聚层不用于减少输入的高度和宽度\n    nn.Conv2d(256, 384, kernel_size=3, padding=1), nn.ReLU(),\n    nn.Conv2d(384, 384, kernel_size=3, padding=1), nn.ReLU(),\n    nn.Conv2d(384, 256, kernel_size=3, padding=1), nn.ReLU(),\n    nn.MaxPool2d(kernel_size=3, stride=2),\n    nn.Flatten(),\n    # 这里，全连接层的输出数量是LeNet中的好几倍。使用dropout层来减轻过拟合\n    nn.Linear(6400, 4096), nn.ReLU(),\n    nn.Dropout(p=0.5),\n    nn.Linear(4096, 4096), nn.ReLU(),\n    nn.Dropout(p=0.5),\n    # 最后是输出层。由于这里使用Fashion-MNIST，所以用类别数为10，而非论文中的1000\n    nn.Linear(4096, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:12:42.410621Z","iopub.execute_input":"2025-06-11T09:12:42.411335Z","iopub.status.idle":"2025-06-11T09:12:50.518850Z","shell.execute_reply.started":"2025-06-11T09:12:42.411299Z","shell.execute_reply":"2025-06-11T09:12:50.518057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.randn(1, 1, 224, 224)\nfor layer in net:\n    X=layer(X)\n    print(layer.__class__.__name__,'output shape:\\t',X.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:12:57.083112Z","iopub.execute_input":"2025-06-11T09:12:57.083930Z","iopub.status.idle":"2025-06-11T09:12:57.202692Z","shell.execute_reply.started":"2025-06-11T09:12:57.083902Z","shell.execute_reply":"2025-06-11T09:12:57.202053Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.1.3. 读取数据集","metadata":{}},{"cell_type":"code","source":"batch_size = 128\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:12:59.399860Z","iopub.execute_input":"2025-06-11T09:12:59.400463Z","iopub.status.idle":"2025-06-11T09:13:04.636550Z","shell.execute_reply.started":"2025-06-11T09:12:59.400438Z","shell.execute_reply":"2025-06-11T09:13:04.635767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.1.4. 训练AlexNet","metadata":{}},{"cell_type":"code","source":"lr, num_epochs = 0.01, 10  # 去掉小数点，使用整数 10\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:13:06.723500Z","iopub.execute_input":"2025-06-11T09:13:06.724333Z","iopub.status.idle":"2025-06-11T09:21:34.380402Z","shell.execute_reply.started":"2025-06-11T09:13:06.724305Z","shell.execute_reply":"2025-06-11T09:21:34.379515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.2. 使用块的网络（VGG）","metadata":{}},{"cell_type":"markdown","source":"### 7.2.1. VGG块","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom d2l import torch as d2l\n\n\ndef vgg_block(num_convs, in_channels, out_channels):\n    layers = []\n    for _ in range(num_convs):\n        layers.append(nn.Conv2d(in_channels, out_channels,\n                                kernel_size=3, padding=1))\n        layers.append(nn.ReLU())\n        in_channels = out_channels\n    layers.append(nn.MaxPool2d(kernel_size=2,stride=2))\n    return nn.Sequential(*layers)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:19.988003Z","iopub.execute_input":"2025-06-11T09:22:19.988765Z","iopub.status.idle":"2025-06-11T09:22:19.993787Z","shell.execute_reply.started":"2025-06-11T09:22:19.988738Z","shell.execute_reply":"2025-06-11T09:22:19.992947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.2.2. VGG网络","metadata":{}},{"cell_type":"code","source":"conv_arch = ((1, 64), (1, 128), (2, 256), (2, 512), (2, 512))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:22.917633Z","iopub.execute_input":"2025-06-11T09:22:22.917901Z","iopub.status.idle":"2025-06-11T09:22:22.922014Z","shell.execute_reply.started":"2025-06-11T09:22:22.917883Z","shell.execute_reply":"2025-06-11T09:22:22.921260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def vgg(conv_arch):\n    conv_blks = []\n    in_channels = 1\n    # 卷积层部分\n    for (num_convs, out_channels) in conv_arch:\n        conv_blks.append(vgg_block(num_convs, in_channels, out_channels))\n        in_channels = out_channels\n\n    return nn.Sequential(\n        *conv_blks, nn.Flatten(),\n        # 全连接层部分\n        nn.Linear(out_channels * 7 * 7, 4096), nn.ReLU(), nn.Dropout(0.5),\n        nn.Linear(4096, 4096), nn.ReLU(), nn.Dropout(0.5),\n        nn.Linear(4096, 10))\n\nnet = vgg(conv_arch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:24.655506Z","iopub.execute_input":"2025-06-11T09:22:24.656220Z","iopub.status.idle":"2025-06-11T09:22:25.749341Z","shell.execute_reply.started":"2025-06-11T09:22:24.656188Z","shell.execute_reply":"2025-06-11T09:22:25.748685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.randn(size=(1, 1, 224, 224))\nfor blk in net:\n    X = blk(X)\n    print(blk.__class__.__name__,'output shape:\\t',X.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:28.062034Z","iopub.execute_input":"2025-06-11T09:22:28.062824Z","iopub.status.idle":"2025-06-11T09:22:28.323724Z","shell.execute_reply.started":"2025-06-11T09:22:28.062791Z","shell.execute_reply":"2025-06-11T09:22:28.322431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.2.3. 训练模型","metadata":{}},{"cell_type":"code","source":"ratio = 4\nsmall_conv_arch = [(pair[0], pair[1] // ratio) for pair in conv_arch]\nnet = vgg(small_conv_arch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:30.543070Z","iopub.execute_input":"2025-06-11T09:22:30.543360Z","iopub.status.idle":"2025-06-11T09:22:30.936524Z","shell.execute_reply.started":"2025-06-11T09:22:30.543340Z","shell.execute_reply":"2025-06-11T09:22:30.935965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 0.05, 10, 128\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:22:32.953535Z","iopub.execute_input":"2025-06-11T09:22:32.954429Z","iopub.status.idle":"2025-06-11T09:37:09.395114Z","shell.execute_reply.started":"2025-06-11T09:22:32.954403Z","shell.execute_reply":"2025-06-11T09:37:09.394177Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.3. 网络中的网络（NiN）","metadata":{}},{"cell_type":"markdown","source":"### 7.3.1. NiN块","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom d2l import torch as d2l\n\n\ndef nin_block(in_channels, out_channels, kernel_size, strides, padding):\n    return nn.Sequential(\n        nn.Conv2d(in_channels, out_channels, kernel_size, strides, padding),\n        nn.ReLU(),\n        nn.Conv2d(out_channels, out_channels, kernel_size=1), nn.ReLU(),\n        nn.Conv2d(out_channels, out_channels, kernel_size=1), nn.ReLU())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:37:09.396725Z","iopub.execute_input":"2025-06-11T09:37:09.397033Z","iopub.status.idle":"2025-06-11T09:37:09.401797Z","shell.execute_reply.started":"2025-06-11T09:37:09.397012Z","shell.execute_reply":"2025-06-11T09:37:09.401170Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.3.2. NiN模型","metadata":{}},{"cell_type":"code","source":"net = nn.Sequential(\n    nin_block(1, 96, kernel_size=11, strides=4, padding=0),\n    nn.MaxPool2d(3, stride=2),\n    nin_block(96, 256, kernel_size=5, strides=1, padding=2),\n    nn.MaxPool2d(3, stride=2),\n    nin_block(256, 384, kernel_size=3, strides=1, padding=1),\n    nn.MaxPool2d(3, stride=2),\n    nn.Dropout(0.5),\n    # 标签类别数是10\n    nin_block(384, 10, kernel_size=3, strides=1, padding=1),\n    nn.AdaptiveAvgPool2d((1, 1)),\n    # 将四维的输出转成二维的输出，其形状为(批量大小,10)\n    nn.Flatten())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:37:09.402383Z","iopub.execute_input":"2025-06-11T09:37:09.402540Z","iopub.status.idle":"2025-06-11T09:37:09.436070Z","shell.execute_reply.started":"2025-06-11T09:37:09.402528Z","shell.execute_reply":"2025-06-11T09:37:09.435571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.rand(size=(1, 1, 224, 224))\nfor layer in net:\n    X = layer(X)\n    print(layer.__class__.__name__,'output shape:\\t', X.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:37:09.437682Z","iopub.execute_input":"2025-06-11T09:37:09.437884Z","iopub.status.idle":"2025-06-11T09:37:09.527387Z","shell.execute_reply.started":"2025-06-11T09:37:09.437863Z","shell.execute_reply":"2025-06-11T09:37:09.526719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 0.1, 10, 128\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:37:09.528289Z","iopub.execute_input":"2025-06-11T09:37:09.528547Z","iopub.status.idle":"2025-06-11T09:48:06.514009Z","shell.execute_reply.started":"2025-06-11T09:37:09.528526Z","shell.execute_reply":"2025-06-11T09:48:06.513185Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.4","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom d2l import torch as d2l\n\n\nclass Inception(nn.Module):\n    # c1--c4是每条路径的输出通道数\n    def __init__(self, in_channels, c1, c2, c3, c4, **kwargs):\n        super(Inception, self).__init__(**kwargs)\n        # 线路1，单1x1卷积层\n        self.p1_1 = nn.Conv2d(in_channels, c1, kernel_size=1)\n        # 线路2，1x1卷积层后接3x3卷积层\n        self.p2_1 = nn.Conv2d(in_channels, c2[0], kernel_size=1)\n        self.p2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=3, padding=1)\n        # 线路3，1x1卷积层后接5x5卷积层\n        self.p3_1 = nn.Conv2d(in_channels, c3[0], kernel_size=1)\n        self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=5, padding=2)\n        # 线路4，3x3最大汇聚层后接1x1卷积层\n        self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)\n        self.p4_2 = nn.Conv2d(in_channels, c4, kernel_size=1)\n\n    def forward(self, x):\n        p1 = F.relu(self.p1_1(x))\n        p2 = F.relu(self.p2_2(F.relu(self.p2_1(x))))\n        p3 = F.relu(self.p3_2(F.relu(self.p3_1(x))))\n        p4 = F.relu(self.p4_2(self.p4_1(x)))\n        # 在通道维度上连结输出\n        return torch.cat((p1, p2, p3, p4), dim=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.515164Z","iopub.execute_input":"2025-06-11T09:48:06.515873Z","iopub.status.idle":"2025-06-11T09:48:06.523689Z","shell.execute_reply.started":"2025-06-11T09:48:06.515848Z","shell.execute_reply":"2025-06-11T09:48:06.523020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n                   nn.ReLU(),\n                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.524641Z","iopub.execute_input":"2025-06-11T09:48:06.524902Z","iopub.status.idle":"2025-06-11T09:48:06.538638Z","shell.execute_reply.started":"2025-06-11T09:48:06.524871Z","shell.execute_reply":"2025-06-11T09:48:06.538061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b2 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=1),\n                   nn.ReLU(),\n                   nn.Conv2d(64, 192, kernel_size=3, padding=1),\n                   nn.ReLU(),\n                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.539536Z","iopub.execute_input":"2025-06-11T09:48:06.539742Z","iopub.status.idle":"2025-06-11T09:48:06.547646Z","shell.execute_reply.started":"2025-06-11T09:48:06.539728Z","shell.execute_reply":"2025-06-11T09:48:06.547160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b3 = nn.Sequential(Inception(192, 64, (96, 128), (16, 32), 32),\n                   Inception(256, 128, (128, 192), (32, 96), 64),\n                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.548811Z","iopub.execute_input":"2025-06-11T09:48:06.549086Z","iopub.status.idle":"2025-06-11T09:48:06.566767Z","shell.execute_reply.started":"2025-06-11T09:48:06.549064Z","shell.execute_reply":"2025-06-11T09:48:06.566293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b4 = nn.Sequential(Inception(480, 192, (96, 208), (16, 48), 64),\n                   Inception(512, 160, (112, 224), (24, 64), 64),\n                   Inception(512, 128, (128, 256), (24, 64), 64),\n                   Inception(512, 112, (144, 288), (32, 64), 64),\n                   Inception(528, 256, (160, 320), (32, 128), 128),\n                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.570514Z","iopub.execute_input":"2025-06-11T09:48:06.571052Z","iopub.status.idle":"2025-06-11T09:48:06.604290Z","shell.execute_reply.started":"2025-06-11T09:48:06.571037Z","shell.execute_reply":"2025-06-11T09:48:06.603807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b5 = nn.Sequential(Inception(832, 256, (160, 320), (32, 128), 128),\n                   Inception(832, 384, (192, 384), (48, 128), 128),\n                   nn.AdaptiveAvgPool2d((1,1)),\n                   nn.Flatten())\n\nnet = nn.Sequential(b1, b2, b3, b4, b5, nn.Linear(1024, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.604858Z","iopub.execute_input":"2025-06-11T09:48:06.605058Z","iopub.status.idle":"2025-06-11T09:48:06.633605Z","shell.execute_reply.started":"2025-06-11T09:48:06.605044Z","shell.execute_reply":"2025-06-11T09:48:06.633158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.rand(size=(1, 1, 96, 96))\nfor layer in net:\n    X = layer(X)\n    print(layer.__class__.__name__,'output shape:\\t', X.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.634228Z","iopub.execute_input":"2025-06-11T09:48:06.634421Z","iopub.status.idle":"2025-06-11T09:48:06.679241Z","shell.execute_reply.started":"2025-06-11T09:48:06.634407Z","shell.execute_reply":"2025-06-11T09:48:06.678543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 0.1, 10, 128\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:48:06.680012Z","iopub.execute_input":"2025-06-11T09:48:06.680253Z","iopub.status.idle":"2025-06-11T09:56:22.772420Z","shell.execute_reply.started":"2025-06-11T09:48:06.680228Z","shell.execute_reply":"2025-06-11T09:56:22.771578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.5","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom d2l import torch as d2l\n\n\ndef batch_norm(X, gamma, beta, moving_mean, moving_var, eps, momentum):\n    # 通过is_grad_enabled来判断当前模式是训练模式还是预测模式\n    if not torch.is_grad_enabled():\n        # 如果是在预测模式下，直接使用传入的移动平均所得的均值和方差\n        X_hat = (X - moving_mean) / torch.sqrt(moving_var + eps)\n    else:\n        assert len(X.shape) in (2, 4)\n        if len(X.shape) == 2:\n            # 使用全连接层的情况，计算特征维上的均值和方差\n            mean = X.mean(dim=0)\n            var = ((X - mean) ** 2).mean(dim=0)\n        else:\n            # 使用二维卷积层的情况，计算通道维上（axis=1）的均值和方差。\n            # 这里我们需要保持X的形状以便后面可以做广播运算\n            mean = X.mean(dim=(0, 2, 3), keepdim=True)\n            var = ((X - mean) ** 2).mean(dim=(0, 2, 3), keepdim=True)\n        # 训练模式下，用当前的均值和方差做标准化\n        X_hat = (X - mean) / torch.sqrt(var + eps)\n        # 更新移动平均的均值和方差\n        moving_mean = momentum * moving_mean + (1.0 - momentum) * mean\n        moving_var = momentum * moving_var + (1.0 - momentum) * var\n    Y = gamma * X_hat + beta  # 缩放和移位\n    return Y, moving_mean.data, moving_var.data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:56:22.773544Z","iopub.execute_input":"2025-06-11T09:56:22.773821Z","iopub.status.idle":"2025-06-11T09:56:22.780990Z","shell.execute_reply.started":"2025-06-11T09:56:22.773798Z","shell.execute_reply":"2025-06-11T09:56:22.780264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BatchNorm(nn.Module):\n    # num_features：完全连接层的输出数量或卷积层的输出通道数。\n    # num_dims：2表示完全连接层，4表示卷积层\n    def __init__(self, num_features, num_dims):\n        super().__init__()\n        if num_dims == 2:\n            shape = (1, num_features)\n        else:\n            shape = (1, num_features, 1, 1)\n        # 参与求梯度和迭代的拉伸和偏移参数，分别初始化成1和0\n        self.gamma = nn.Parameter(torch.ones(shape))\n        self.beta = nn.Parameter(torch.zeros(shape))\n        # 非模型参数的变量初始化为0和1\n        self.moving_mean = torch.zeros(shape)\n        self.moving_var = torch.ones(shape)\n\n    def forward(self, X):\n        # 如果X不在内存上，将moving_mean和moving_var\n        # 复制到X所在显存上\n        if self.moving_mean.device != X.device:\n            self.moving_mean = self.moving_mean.to(X.device)\n            self.moving_var = self.moving_var.to(X.device)\n        # 保存更新过的moving_mean和moving_var\n        Y, self.moving_mean, self.moving_var = batch_norm(\n            X, self.gamma, self.beta, self.moving_mean,\n            self.moving_var, eps=1e-5, momentum=0.9)\n        return Y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:56:22.781802Z","iopub.execute_input":"2025-06-11T09:56:22.782383Z","iopub.status.idle":"2025-06-11T09:56:22.796366Z","shell.execute_reply.started":"2025-06-11T09:56:22.782364Z","shell.execute_reply":"2025-06-11T09:56:22.795679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net = nn.Sequential(\n    nn.Conv2d(1, 6, kernel_size=5), BatchNorm(6, num_dims=4), nn.Sigmoid(),\n    nn.AvgPool2d(kernel_size=2, stride=2),\n    nn.Conv2d(6, 16, kernel_size=5), BatchNorm(16, num_dims=4), nn.Sigmoid(),\n    nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n    nn.Linear(16*4*4, 120), BatchNorm(120, num_dims=2), nn.Sigmoid(),\n    nn.Linear(120, 84), BatchNorm(84, num_dims=2), nn.Sigmoid(),\n    nn.Linear(84, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:56:22.797156Z","iopub.execute_input":"2025-06-11T09:56:22.797468Z","iopub.status.idle":"2025-06-11T09:56:22.811628Z","shell.execute_reply.started":"2025-06-11T09:56:22.797446Z","shell.execute_reply":"2025-06-11T09:56:22.810993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 1.0, 10, 256\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:56:22.812478Z","iopub.execute_input":"2025-06-11T09:56:22.812724Z","iopub.status.idle":"2025-06-11T09:57:15.680512Z","shell.execute_reply.started":"2025-06-11T09:56:22.812704Z","shell.execute_reply":"2025-06-11T09:57:15.679790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net[1].gamma.reshape((-1,)), net[1].beta.reshape((-1,))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:57:15.681463Z","iopub.execute_input":"2025-06-11T09:57:15.681689Z","iopub.status.idle":"2025-06-11T09:57:15.873396Z","shell.execute_reply.started":"2025-06-11T09:57:15.681668Z","shell.execute_reply":"2025-06-11T09:57:15.872745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net = nn.Sequential(\n    nn.Conv2d(1, 6, kernel_size=5), nn.BatchNorm2d(6), nn.Sigmoid(),\n    nn.AvgPool2d(kernel_size=2, stride=2),\n    nn.Conv2d(6, 16, kernel_size=5), nn.BatchNorm2d(16), nn.Sigmoid(),\n    nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n    nn.Linear(256, 120), nn.BatchNorm1d(120), nn.Sigmoid(),\n    nn.Linear(120, 84), nn.BatchNorm1d(84), nn.Sigmoid(),\n    nn.Linear(84, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:57:15.874018Z","iopub.execute_input":"2025-06-11T09:57:15.874229Z","iopub.status.idle":"2025-06-11T09:57:15.882506Z","shell.execute_reply.started":"2025-06-11T09:57:15.874213Z","shell.execute_reply":"2025-06-11T09:57:15.881796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:57:15.883226Z","iopub.execute_input":"2025-06-11T09:57:15.883439Z","iopub.status.idle":"2025-06-11T09:58:01.705460Z","shell.execute_reply.started":"2025-06-11T09:57:15.883417Z","shell.execute_reply":"2025-06-11T09:58:01.704758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.6","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom d2l import torch as d2l\n\n\nclass Residual(nn.Module):  #@save\n    def __init__(self, input_channels, num_channels,\n                 use_1x1conv=False, strides=1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(input_channels, num_channels,\n                               kernel_size=3, padding=1, stride=strides)\n        self.conv2 = nn.Conv2d(num_channels, num_channels,\n                               kernel_size=3, padding=1)\n        if use_1x1conv:\n            self.conv3 = nn.Conv2d(input_channels, num_channels,\n                                   kernel_size=1, stride=strides)\n        else:\n            self.conv3 = None\n        self.bn1 = nn.BatchNorm2d(num_channels)\n        self.bn2 = nn.BatchNorm2d(num_channels)\n\n    def forward(self, X):\n        Y = F.relu(self.bn1(self.conv1(X)))\n        Y = self.bn2(self.conv2(Y))\n        if self.conv3:\n            X = self.conv3(X)\n        Y += X\n        return F.relu(Y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.706616Z","iopub.execute_input":"2025-06-11T09:58:01.707364Z","iopub.status.idle":"2025-06-11T09:58:01.714247Z","shell.execute_reply.started":"2025-06-11T09:58:01.707337Z","shell.execute_reply":"2025-06-11T09:58:01.713174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blk = Residual(3,3)\nX = torch.rand(4, 3, 6, 6)\nY = blk(X)\nY.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.714991Z","iopub.execute_input":"2025-06-11T09:58:01.715323Z","iopub.status.idle":"2025-06-11T09:58:01.738292Z","shell.execute_reply.started":"2025-06-11T09:58:01.715305Z","shell.execute_reply":"2025-06-11T09:58:01.737608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blk = Residual(3,6, use_1x1conv=True, strides=2)\nblk(X).shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.739155Z","iopub.execute_input":"2025-06-11T09:58:01.739371Z","iopub.status.idle":"2025-06-11T09:58:01.747807Z","shell.execute_reply.started":"2025-06-11T09:58:01.739356Z","shell.execute_reply":"2025-06-11T09:58:01.747102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n                   nn.BatchNorm2d(64), nn.ReLU(),\n                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.748729Z","iopub.execute_input":"2025-06-11T09:58:01.749153Z","iopub.status.idle":"2025-06-11T09:58:01.754056Z","shell.execute_reply.started":"2025-06-11T09:58:01.749110Z","shell.execute_reply":"2025-06-11T09:58:01.753486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resnet_block(input_channels, num_channels, num_residuals,\n                 first_block=False):\n    blk = []\n    for i in range(num_residuals):\n        if i == 0 and not first_block:\n            blk.append(Residual(input_channels, num_channels,\n                                use_1x1conv=True, strides=2))\n        else:\n            blk.append(Residual(num_channels, num_channels))\n    return blk","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.754716Z","iopub.execute_input":"2025-06-11T09:58:01.754932Z","iopub.status.idle":"2025-06-11T09:58:01.765515Z","shell.execute_reply.started":"2025-06-11T09:58:01.754916Z","shell.execute_reply":"2025-06-11T09:58:01.764734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b2 = nn.Sequential(*resnet_block(64, 64, 2, first_block=True))\nb3 = nn.Sequential(*resnet_block(64, 128, 2))\nb4 = nn.Sequential(*resnet_block(128, 256, 2))\nb5 = nn.Sequential(*resnet_block(256, 512, 2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.766349Z","iopub.execute_input":"2025-06-11T09:58:01.766676Z","iopub.status.idle":"2025-06-11T09:58:01.879576Z","shell.execute_reply.started":"2025-06-11T09:58:01.766655Z","shell.execute_reply":"2025-06-11T09:58:01.879007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net = nn.Sequential(b1, b2, b3, b4, b5,\n                    nn.AdaptiveAvgPool2d((1,1)),\n                    nn.Flatten(), nn.Linear(512, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.880689Z","iopub.execute_input":"2025-06-11T09:58:01.880939Z","iopub.status.idle":"2025-06-11T09:58:01.885639Z","shell.execute_reply.started":"2025-06-11T09:58:01.880914Z","shell.execute_reply":"2025-06-11T09:58:01.885099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.rand(size=(1, 1, 224, 224))\nfor layer in net:\n    X = layer(X)\n    print(layer.__class__.__name__,'output shape:\\t', X.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.888984Z","iopub.execute_input":"2025-06-11T09:58:01.889244Z","iopub.status.idle":"2025-06-11T09:58:01.966908Z","shell.execute_reply.started":"2025-06-11T09:58:01.889218Z","shell.execute_reply":"2025-06-11T09:58:01.966270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 0.05, 10, 256\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T09:58:01.967548Z","iopub.execute_input":"2025-06-11T09:58:01.967749Z","iopub.status.idle":"2025-06-11T10:04:49.473010Z","shell.execute_reply.started":"2025-06-11T09:58:01.967734Z","shell.execute_reply":"2025-06-11T10:04:49.472087Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.7","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom d2l import torch as d2l\n\n\ndef conv_block(input_channels, num_channels):\n    return nn.Sequential(\n        nn.BatchNorm2d(input_channels), nn.ReLU(),\n        nn.Conv2d(input_channels, num_channels, kernel_size=3, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.474149Z","iopub.execute_input":"2025-06-11T10:04:49.474803Z","iopub.status.idle":"2025-06-11T10:04:49.478980Z","shell.execute_reply.started":"2025-06-11T10:04:49.474778Z","shell.execute_reply":"2025-06-11T10:04:49.478376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DenseBlock(nn.Module):\n    def __init__(self, num_convs, input_channels, num_channels):\n        super(DenseBlock, self).__init__()\n        layer = []\n        for i in range(num_convs):\n            layer.append(conv_block(\n                num_channels * i + input_channels, num_channels))\n        self.net = nn.Sequential(*layer)\n\n    def forward(self, X):\n        for blk in self.net:\n            Y = blk(X)\n            # 连接通道维度上每个块的输入和输出\n            X = torch.cat((X, Y), dim=1)\n        return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.479880Z","iopub.execute_input":"2025-06-11T10:04:49.480168Z","iopub.status.idle":"2025-06-11T10:04:49.492531Z","shell.execute_reply.started":"2025-06-11T10:04:49.480143Z","shell.execute_reply":"2025-06-11T10:04:49.491882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blk = DenseBlock(2, 3, 10)\nX = torch.randn(4, 3, 8, 8)\nY = blk(X)\nY.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.493390Z","iopub.execute_input":"2025-06-11T10:04:49.493636Z","iopub.status.idle":"2025-06-11T10:04:49.512594Z","shell.execute_reply.started":"2025-06-11T10:04:49.493610Z","shell.execute_reply":"2025-06-11T10:04:49.511739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transition_block(input_channels, num_channels):\n    return nn.Sequential(\n        nn.BatchNorm2d(input_channels), nn.ReLU(),\n        nn.Conv2d(input_channels, num_channels, kernel_size=1),\n        nn.AvgPool2d(kernel_size=2, stride=2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.513263Z","iopub.execute_input":"2025-06-11T10:04:49.513453Z","iopub.status.idle":"2025-06-11T10:04:49.517404Z","shell.execute_reply.started":"2025-06-11T10:04:49.513440Z","shell.execute_reply":"2025-06-11T10:04:49.516590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blk = transition_block(23, 10)\nblk(Y).shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.518154Z","iopub.execute_input":"2025-06-11T10:04:49.518378Z","iopub.status.idle":"2025-06-11T10:04:49.536841Z","shell.execute_reply.started":"2025-06-11T10:04:49.518356Z","shell.execute_reply":"2025-06-11T10:04:49.536183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b1 = nn.Sequential(\n    nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n    nn.BatchNorm2d(64), nn.ReLU(),\n    nn.MaxPool2d(kernel_size=3, stride=2, padding=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.537728Z","iopub.execute_input":"2025-06-11T10:04:49.538327Z","iopub.status.idle":"2025-06-11T10:04:49.548900Z","shell.execute_reply.started":"2025-06-11T10:04:49.538308Z","shell.execute_reply":"2025-06-11T10:04:49.548230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# num_channels为当前的通道数\nnum_channels, growth_rate = 64, 32\nnum_convs_in_dense_blocks = [4, 4, 4, 4]\nblks = []\nfor i, num_convs in enumerate(num_convs_in_dense_blocks):\n    blks.append(DenseBlock(num_convs, num_channels, growth_rate))\n    # 上一个稠密块的输出通道数\n    num_channels += num_convs * growth_rate\n    # 在稠密块之间添加一个转换层，使通道数量减半\n    if i != len(num_convs_in_dense_blocks) - 1:\n        blks.append(transition_block(num_channels, num_channels // 2))\n        num_channels = num_channels // 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.549804Z","iopub.execute_input":"2025-06-11T10:04:49.550049Z","iopub.status.idle":"2025-06-11T10:04:49.574361Z","shell.execute_reply.started":"2025-06-11T10:04:49.550034Z","shell.execute_reply":"2025-06-11T10:04:49.573858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net = nn.Sequential(\n    b1, *blks,\n    nn.BatchNorm2d(num_channels), nn.ReLU(),\n    nn.AdaptiveAvgPool2d((1, 1)),\n    nn.Flatten(),\n    nn.Linear(num_channels, 10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.575109Z","iopub.execute_input":"2025-06-11T10:04:49.575360Z","iopub.status.idle":"2025-06-11T10:04:49.580146Z","shell.execute_reply.started":"2025-06-11T10:04:49.575338Z","shell.execute_reply":"2025-06-11T10:04:49.579478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr, num_epochs, batch_size = 0.1, 10, 256\ntrain_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)\nd2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-11T10:04:49.580798Z","iopub.execute_input":"2025-06-11T10:04:49.580982Z","iopub.status.idle":"2025-06-11T10:10:53.107669Z","shell.execute_reply.started":"2025-06-11T10:04:49.580969Z","shell.execute_reply":"2025-06-11T10:10:53.106915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}