{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 图像分类\n\n何秋楠","metadata":{}},{"cell_type":"markdown","source":"## 1.数据准备","metadata":{}},{"cell_type":"code","source":"!pip install d2l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:16:51.149122Z","iopub.execute_input":"2025-06-25T01:16:51.149416Z","iopub.status.idle":"2025-06-25T01:17:18.907828Z","shell.execute_reply.started":"2025-06-25T01:16:51.149367Z","shell.execute_reply":"2025-06-25T01:17:18.906760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import collections\nimport math\nimport os\nimport shutil\nimport pandas as pd\nimport torch\nimport torchvision\nfrom torch import nn\nfrom d2l import torch as d2l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:18.908964Z","iopub.execute_input":"2025-06-25T01:17:18.909288Z","iopub.status.idle":"2025-06-25T01:17:30.306481Z","shell.execute_reply.started":"2025-06-25T01:17:18.909258Z","shell.execute_reply":"2025-06-25T01:17:30.305477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#@save\nd2l.DATA_HUB['cifar10_tiny'] = (d2l.DATA_URL + 'kaggle_cifar10_tiny.zip',\n                                '2068874e4b9a9f0fb07ebe0ad2b29754449ccacd')\n\n# 如果使用完整的Kaggle竞赛的数据集，设置demo为False\ndemo = True\n\nif demo:\n    data_dir = d2l.download_extract('cifar10_tiny')\nelse:\n    data_dir = '../data/cifar-10/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:30.308402Z","iopub.execute_input":"2025-06-25T01:17:30.308822Z","iopub.status.idle":"2025-06-25T01:17:31.045202Z","shell.execute_reply.started":"2025-06-25T01:17:30.308800Z","shell.execute_reply":"2025-06-25T01:17:31.044483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## 整理数据集","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.046114Z","iopub.execute_input":"2025-06-25T01:17:31.046435Z","iopub.status.idle":"2025-06-25T01:17:31.050516Z","shell.execute_reply.started":"2025-06-25T01:17:31.046407Z","shell.execute_reply":"2025-06-25T01:17:31.049575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#@save\ndef read_csv_labels(fname):\n    \"\"\"读取fname来给标签字典返回一个文件名\"\"\"\n    with open(fname, 'r') as f:\n        # 跳过文件头行(列名)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))\nprint('# 训练样本 :', len(labels))\nprint('# 类别 :', len(set(labels.values())))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.051353Z","iopub.execute_input":"2025-06-25T01:17:31.051717Z","iopub.status.idle":"2025-06-25T01:17:31.073147Z","shell.execute_reply.started":"2025-06-25T01:17:31.051690Z","shell.execute_reply":"2025-06-25T01:17:31.072184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#@save\ndef copyfile(filename, target_dir):\n    \"\"\"将文件复制到目标目录\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\n#@save\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"将验证集从原始的训练集中拆分出来\"\"\"\n    # 训练数据集中样本最少的类别中的样本数\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # 验证集中每个类别的样本数\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.074103Z","iopub.execute_input":"2025-06-25T01:17:31.074448Z","iopub.status.idle":"2025-06-25T01:17:31.089334Z","shell.execute_reply.started":"2025-06-25T01:17:31.074421Z","shell.execute_reply":"2025-06-25T01:17:31.088418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#@save\ndef reorg_test(data_dir):\n    \"\"\"在预测期间整理测试集，以方便读取\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.090231Z","iopub.execute_input":"2025-06-25T01:17:31.090564Z","iopub.status.idle":"2025-06-25T01:17:31.112154Z","shell.execute_reply.started":"2025-06-25T01:17:31.090534Z","shell.execute_reply":"2025-06-25T01:17:31.111309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.113109Z","iopub.execute_input":"2025-06-25T01:17:31.113805Z","iopub.status.idle":"2025-06-25T01:17:31.128807Z","shell.execute_reply.started":"2025-06-25T01:17:31.113772Z","shell.execute_reply":"2025-06-25T01:17:31.127813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch_size = 32 if demo else 128\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.132225Z","iopub.execute_input":"2025-06-25T01:17:31.132566Z","iopub.status.idle":"2025-06-25T01:17:31.463356Z","shell.execute_reply.started":"2025-06-25T01:17:31.132544Z","shell.execute_reply":"2025-06-25T01:17:31.462636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##图像增广","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.464244Z","iopub.execute_input":"2025-06-25T01:17:31.464495Z","iopub.status.idle":"2025-06-25T01:17:31.468820Z","shell.execute_reply.started":"2025-06-25T01:17:31.464477Z","shell.execute_reply":"2025-06-25T01:17:31.467666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    # 在高度和宽度上将图像放大到40像素的正方形\n    torchvision.transforms.Resize(40),\n    # 随机裁剪出一个高度和宽度均为40像素的正方形图像，\n    # 生成一个面积为原始图像面积0.64～1倍的小正方形，\n    # 然后将其缩放为高度和宽度均为32像素的正方形\n    torchvision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0),\n                                                   ratio=(1.0, 1.0)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    # 标准化图像的每个通道\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.469764Z","iopub.execute_input":"2025-06-25T01:17:31.470075Z","iopub.status.idle":"2025-06-25T01:17:31.491489Z","shell.execute_reply.started":"2025-06-25T01:17:31.470036Z","shell.execute_reply":"2025-06-25T01:17:31.490781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.492494Z","iopub.execute_input":"2025-06-25T01:17:31.492799Z","iopub.status.idle":"2025-06-25T01:17:31.510884Z","shell.execute_reply.started":"2025-06-25T01:17:31.492779Z","shell.execute_reply":"2025-06-25T01:17:31.509847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##读取数据集","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.511858Z","iopub.execute_input":"2025-06-25T01:17:31.512830Z","iopub.status.idle":"2025-06-25T01:17:31.527236Z","shell.execute_reply.started":"2025-06-25T01:17:31.512686Z","shell.execute_reply":"2025-06-25T01:17:31.526307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.528175Z","iopub.execute_input":"2025-06-25T01:17:31.528587Z","iopub.status.idle":"2025-06-25T01:17:31.553578Z","shell.execute_reply.started":"2025-06-25T01:17:31.528552Z","shell.execute_reply":"2025-06-25T01:17:31.552702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.554394Z","iopub.execute_input":"2025-06-25T01:17:31.554712Z","iopub.status.idle":"2025-06-25T01:17:31.561837Z","shell.execute_reply.started":"2025-06-25T01:17:31.554682Z","shell.execute_reply":"2025-06-25T01:17:31.561151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##定义模型","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.562998Z","iopub.execute_input":"2025-06-25T01:17:31.563286Z","iopub.status.idle":"2025-06-25T01:17:31.578358Z","shell.execute_reply.started":"2025-06-25T01:17:31.563257Z","shell.execute_reply":"2025-06-25T01:17:31.577397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_net():\n    num_classes = 10\n    net = d2l.resnet18(num_classes, 3)\n    return net\n\nloss = nn.CrossEntropyLoss(reduction=\"none\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.579409Z","iopub.execute_input":"2025-06-25T01:17:31.579663Z","iopub.status.idle":"2025-06-25T01:17:31.595579Z","shell.execute_reply.started":"2025-06-25T01:17:31.579640Z","shell.execute_reply":"2025-06-25T01:17:31.594592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##定义训练函数","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.596491Z","iopub.execute_input":"2025-06-25T01:17:31.596726Z","iopub.status.idle":"2025-06-25T01:17:31.611278Z","shell.execute_reply.started":"2025-06-25T01:17:31.596709Z","shell.execute_reply":"2025-06-25T01:17:31.610541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,\n          lr_decay):\n    trainer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9,\n                              weight_decay=wd)\n    scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay)\n    num_batches, timer = len(train_iter), d2l.Timer()\n    legend = ['train loss', 'train acc']\n    if valid_iter is not None:\n        legend.append('valid acc')\n    animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],\n                            legend=legend)\n    net = nn.DataParallel(net, device_ids=devices).to(devices[0])\n    for epoch in range(num_epochs):\n        net.train()\n        metric = d2l.Accumulator(3)\n        for i, (features, labels) in enumerate(train_iter):\n            timer.start()\n            l, acc = d2l.train_batch_ch13(net, features, labels,\n                                          loss, trainer, devices)\n            metric.add(l, acc, labels.shape[0])\n            timer.stop()\n            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n                animator.add(epoch + (i + 1) / num_batches,\n                             (metric[0] / metric[2], metric[1] / metric[2],\n                              None))\n        if valid_iter is not None:\n            valid_acc = d2l.evaluate_accuracy_gpu(net, valid_iter)\n            animator.add(epoch + 1, (None, None, valid_acc))\n        scheduler.step()\n    measures = (f'train loss {metric[0] / metric[2]:.3f}, '\n                f'train acc {metric[1] / metric[2]:.3f}')\n    if valid_iter is not None:\n        measures += f', valid acc {valid_acc:.3f}'\n    print(measures + f'\\n{metric[2] * num_epochs / timer.sum():.1f}'\n          f' examples/sec on {str(devices)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.612169Z","iopub.execute_input":"2025-06-25T01:17:31.612440Z","iopub.status.idle":"2025-06-25T01:17:31.627832Z","shell.execute_reply.started":"2025-06-25T01:17:31.612415Z","shell.execute_reply":"2025-06-25T01:17:31.627065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##训练和验证模型","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.628672Z","iopub.execute_input":"2025-06-25T01:17:31.628959Z","iopub.status.idle":"2025-06-25T01:17:31.645622Z","shell.execute_reply.started":"2025-06-25T01:17:31.628934Z","shell.execute_reply":"2025-06-25T01:17:31.644717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"devices = ['cpu']  # 强制使用CPU\nnum_epochs, lr, wd = 20, 2e-4, 5e-4\nlr_period, lr_decay, net = 4, 0.9, get_net()\n\ntrain(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period, lr_decay)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:17:31.646600Z","iopub.execute_input":"2025-06-25T01:17:31.646908Z","iopub.status.idle":"2025-06-25T01:29:57.361641Z","shell.execute_reply.started":"2025-06-25T01:17:31.646882Z","shell.execute_reply":"2025-06-25T01:29:57.360830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##在 Kaggle 上对测试集进行分类并提交结果","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:29:57.362645Z","iopub.execute_input":"2025-06-25T01:29:57.362987Z","iopub.status.idle":"2025-06-25T01:29:57.367416Z","shell.execute_reply.started":"2025-06-25T01:29:57.362959Z","shell.execute_reply":"2025-06-25T01:29:57.366492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"net, preds = get_net(), []\ntrain(net, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period,\n      lr_decay)\n\nfor X, _ in test_iter:\n    y_hat = net(X.to(devices[0]))\n    preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())\nsorted_ids = list(range(1, len(test_ds) + 1))\nsorted_ids.sort(key=lambda x: str(x))\ndf = pd.DataFrame({'id': sorted_ids, 'label': preds})\ndf['label'] = df['label'].apply(lambda x: train_valid_ds.classes[x])\ndf.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T01:29:57.368321Z","iopub.execute_input":"2025-06-25T01:29:57.368661Z","iopub.status.idle":"2025-06-25T01:43:25.664287Z","shell.execute_reply.started":"2025-06-25T01:29:57.368636Z","shell.execute_reply":"2025-06-25T01:43:25.663356Z"}},"outputs":[],"execution_count":null}]}