{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.263529Z","iopub.execute_input":"2023-04-09T09:48:20.263917Z","iopub.status.idle":"2023-04-09T09:48:20.268916Z","shell.execute_reply.started":"2023-04-09T09:48:20.263881Z","shell.execute_reply":"2023-04-09T09:48:20.267821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.DataFrame({'id': [1,2], 'label': [2,3]})\n# df['label'] = df['label'].apply(lambda x: [x])\n# df.to_csv('./submission_test.csv', index=False)\n# print(\"1\")","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.278720Z","iopub.execute_input":"2023-04-09T09:48:20.279014Z","iopub.status.idle":"2023-04-09T09:48:20.288932Z","shell.execute_reply.started":"2023-04-09T09:48:20.278987Z","shell.execute_reply":"2023-04-09T09:48:20.287721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install d2l","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.291670Z","iopub.execute_input":"2023-04-09T09:48:20.292164Z","iopub.status.idle":"2023-04-09T09:48:20.297410Z","shell.execute_reply.started":"2023-04-09T09:48:20.292125Z","shell.execute_reply":"2023-04-09T09:48:20.296005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nimport math\nimport os\nimport shutil\nimport pandas as pd\nimport torch\nimport torchvision\nfrom d2l import torch as d2l\nfrom torch import nn","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.303974Z","iopub.execute_input":"2023-04-09T09:48:20.304699Z","iopub.status.idle":"2023-04-09T09:48:20.310424Z","shell.execute_reply.started":"2023-04-09T09:48:20.304661Z","shell.execute_reply":"2023-04-09T09:48:20.309128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.DataFrame({'id': [1,2], 'label': [2,3]})\n# df['label'] = df['label'].apply(lambda x: [x])\n# df.to_csv('./submission_test.csv', index=False)\n# print(\"1\")","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.312768Z","iopub.execute_input":"2023-04-09T09:48:20.313685Z","iopub.status.idle":"2023-04-09T09:48:20.323731Z","shell.execute_reply.started":"2023-04-09T09:48:20.313648Z","shell.execute_reply":"2023-04-09T09:48:20.322550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 小数据集测试\ncifar10_tiny_dir = \"/kaggle/input/cifar10-tiny/kaggle_cifar10_tiny\"\ncifar10_un7zip_dir = \"/kaggle/input/cifar10-un7zip\"\n\nwork_dir = \"/kaggle/working/\"\n\n\n\ndemo = False\n\ndata_dir = cifar10_un7zip_dir\ndata_test_dir = \"test/test\"\ndata_train_dir= \"train/train\"\n# /kaggle/input/cifar10-un7zip/test/test\nif demo:\n    data_dir = cifar10_tiny_dir\n    data_test_dir =\"test\"\n    data_train_dir= \"train\"","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.343199Z","iopub.execute_input":"2023-04-09T09:48:20.343482Z","iopub.status.idle":"2023-04-09T09:48:20.348812Z","shell.execute_reply.started":"2023-04-09T09:48:20.343456Z","shell.execute_reply":"2023-04-09T09:48:20.347681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clear_folder(folder_path):\n    for filename in os.listdir(folder_path):\n        file_path = os.path.join(folder_path, filename)\n        try:\n            if os.path.isfile(file_path) or os.path.islink(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                shutil.rmtree(file_path)\n        except Exception as e:\n            print('无法删除 %s. 原因: %s' % (file_path, e))\n\nclear_folder(\"/kaggle/working/\")","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.359643Z","iopub.execute_input":"2023-04-09T09:48:20.360178Z","iopub.status.idle":"2023-04-09T09:48:20.391634Z","shell.execute_reply.started":"2023-04-09T09:48:20.360150Z","shell.execute_reply":"2023-04-09T09:48:20.390712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Accumulator:\n    \"\"\"For accumulating sums over `n` variables.\"\"\"\n    def __init__(self, n):\n        \"\"\"Defined in :numref:`sec_softmax_scratch`\"\"\"\n        self.data = [0.0] * n\n\n    def add(self, *args):\n        self.data = [a + float(b) for a, b in zip(self.data, args)]\n\n    def reset(self):\n        self.data = [0.0] * len(self.data)\n\n    def __getitem__(self, idx):\n        return self.data[idx]\n\ndef resnet18(num_classes, in_channels=1):\n    \"\"\"A slightly modified ResNet-18 model.\n\n    Defined in :numref:`sec_multi_gpu_concise`\"\"\"\n    def resnet_block(in_channels, out_channels, num_residuals,\n                     first_block=False):\n        blk = []\n        for i in range(num_residuals):\n            if i == 0 and not first_block:\n                blk.append(d2l.Residual(in_channels, out_channels,\n                                        use_1x1conv=True, strides=2))\n            else:\n                blk.append(d2l.Residual(out_channels, out_channels))\n        return nn.Sequential(*blk)\n\n    # This model uses a smaller convolution kernel, stride, and padding and\n    # removes the maximum pooling layer\n    net = nn.Sequential(\n        nn.Conv2d(in_channels, 64, kernel_size=3, stride=1, padding=1),\n        nn.BatchNorm2d(64),\n        nn.ReLU())\n    net.add_module(\"resnet_block1\", resnet_block(64, 64, 2, first_block=True))\n    net.add_module(\"resnet_block2\", resnet_block(64, 128, 2))\n    net.add_module(\"resnet_block3\", resnet_block(128, 256, 2))\n    net.add_module(\"resnet_block4\", resnet_block(256, 512, 2))\n    net.add_module(\"global_avg_pool\", nn.AdaptiveAvgPool2d((1,1)))\n    net.add_module(\"fc\", nn.Sequential(nn.Flatten(),\n                                       nn.Linear(512, num_classes)))\n    return net\n","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.409194Z","iopub.execute_input":"2023-04-09T09:48:20.409504Z","iopub.status.idle":"2023-04-09T09:48:20.420949Z","shell.execute_reply.started":"2023-04-09T09:48:20.409465Z","shell.execute_reply":"2023-04-09T09:48:20.419692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"读取fname来给标签字典返回一个文件名\"\"\"\n    with open(fname, 'r') as f:\n        # 跳过文件头行(列名)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))\nprint('# 训练样本 :', len(labels))\nprint('# 类别 :', len(set(labels.values())))","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.441191Z","iopub.execute_input":"2023-04-09T09:48:20.441734Z","iopub.status.idle":"2023-04-09T09:48:20.499220Z","shell.execute_reply.started":"2023-04-09T09:48:20.441698Z","shell.execute_reply":"2023-04-09T09:48:20.498025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"将文件复制到目标目录\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\n# valid_radio 验证集中的样本数与原始训练集中的样本数之比\ndef reorg_train_valid(data_dir,work_dir,data_test_dir, labels, valid_ratio):\n    \"\"\"将验证集从原始的训练集中拆分出来\"\"\"\n    # 训练数据集中样本最少的类别中的样本数\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # 验证集中每个类别的样本数\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count_map = {}\n    for train_file in os.listdir(os.path.join(data_dir, data_train_dir)):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir,data_train_dir, train_file)\n        copyfile(fname, os.path.join(work_dir, 'train_valid_test',\n                                     'train_valid', label))\n#         print(fname)\n        if label not in label_count_map or label_count_map[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(work_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count_map[label] = label_count_map.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(work_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.501218Z","iopub.execute_input":"2023-04-09T09:48:20.502157Z","iopub.status.idle":"2023-04-09T09:48:20.512988Z","shell.execute_reply.started":"2023-04-09T09:48:20.502116Z","shell.execute_reply":"2023-04-09T09:48:20.511708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_test(data_dir,work_dir,data_test_dir):\n    \"\"\"在预测期间整理测试集，以方便读取\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, data_test_dir)):\n        copyfile(os.path.join(data_dir, data_test_dir, test_file),\n                 os.path.join(work_dir, 'train_valid_test', 'test',\n                              'unknown'))\n# reorg_test(data_dir,work_dir,data_test_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_cifar10_data(data_dir,work_dir,data_test_dir, valid_ratio):\n    labels = read_csv_labels(os.path.join(data_dir, 'trainLabels.csv'))\n    reorg_train_valid(data_dir,work_dir, data_test_dir,labels, valid_ratio)\n    reorg_test(data_dir,work_dir,data_test_dir)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:48:20.525341Z","iopub.execute_input":"2023-04-09T09:48:20.526020Z","iopub.status.idle":"2023-04-09T09:48:20.533870Z","shell.execute_reply.started":"2023-04-09T09:48:20.525982Z","shell.execute_reply":"2023-04-09T09:48:20.533106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch_size = 32\nbatch_size = 32 if demo else 128\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir,work_dir,data_test_dir, valid_ratio)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    # 在高度和宽度上将图像放大到40像素的正方形\n    torchvision.transforms.Resize(40),\n    # 随机裁剪出一个高度和宽度均为40像素的正方形图像，\n    # 生成一个面积为原始图像面积0.64～1倍的小正方形，\n    # 然后将其缩放为高度和宽度均为32像素的正方形\n    torchvision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0),\n                                                   ratio=(1.0, 1.0)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    # 标准化图像的每个通道\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.296108Z","iopub.status.idle":"2023-04-09T09:49:05.296462Z","shell.execute_reply.started":"2023-04-09T09:49:05.296289Z","shell.execute_reply":"2023-04-09T09:49:05.296306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.299003Z","iopub.status.idle":"2023-04-09T09:49:05.299520Z","shell.execute_reply.started":"2023-04-09T09:49:05.299278Z","shell.execute_reply":"2023-04-09T09:49:05.299303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(work_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(work_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.301261Z","iopub.status.idle":"2023-04-09T09:49:05.301744Z","shell.execute_reply.started":"2023-04-09T09:49:05.301495Z","shell.execute_reply":"2023-04-09T09:49:05.301520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.303828Z","iopub.status.idle":"2023-04-09T09:49:05.305431Z","shell.execute_reply.started":"2023-04-09T09:49:05.305155Z","shell.execute_reply":"2023-04-09T09:49:05.305182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_net():\n    num_classes = 10\n    net = d2l.resnet18(num_classes, 3)\n    return net\n\nloss = nn.CrossEntropyLoss(reduction=\"none\")","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.306905Z","iopub.status.idle":"2023-04-09T09:49:05.307800Z","shell.execute_reply.started":"2023-04-09T09:49:05.307537Z","shell.execute_reply":"2023-04-09T09:49:05.307563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_batch(net, X, y, loss, trainer, devices):\n    \"\"\"Train for a minibatch with mutiple GPUs (defined in Chapter 13).\n\n    Defined in :numref:`sec_image_augmentation`\"\"\"\n    if isinstance(X, list):\n        # Required for BERT fine-tuning (to be covered later)\n        X = [x.to(devices[0]) for x in X]\n    else:\n        X = X.to(devices[0])\n    y = y.to(devices[0])\n    net.train()\n    trainer.zero_grad()\n    pred = net(X)\n    l = loss(pred, y)\n    l.sum().backward()\n    trainer.step()\n    train_loss_sum = l.sum()\n    train_acc_sum = d2l.accuracy(pred, y)\n    return train_loss_sum, train_acc_sum","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.309293Z","iopub.status.idle":"2023-04-09T09:49:05.310141Z","shell.execute_reply.started":"2023-04-09T09:49:05.309863Z","shell.execute_reply":"2023-04-09T09:49:05.309889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,\n          lr_decay):\n    trainer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9,\n                              weight_decay=wd)\n    scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay)\n    num_batches, timer = len(train_iter), d2l.Timer()\n    legend = ['train loss', 'train acc']\n    if valid_iter is not None:\n        legend.append('valid acc')\n    animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],\n                            legend=legend)\n    net = nn.DataParallel(net, device_ids=devices).to(devices[0])\n    for epoch in range(num_epochs):\n        net.train()\n        metric = d2l.Accumulator(3)\n        for i, (features, labels) in enumerate(train_iter):\n            timer.start()\n            l, acc = train_batch(net, features, labels,\n                                          loss, trainer, devices)\n            metric.add(l, acc, labels.shape[0])\n            timer.stop()\n            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n                animator.add(epoch + (i + 1) / num_batches,\n                             (metric[0] / metric[2], metric[1] / metric[2],\n                              None))\n        if valid_iter is not None:\n            valid_acc = d2l.evaluate_accuracy_gpu(net, valid_iter)\n            animator.add(epoch + 1, (None, None, valid_acc))\n        scheduler.step()\n    measures = (f'train loss {metric[0] / metric[2]:.3f}, '\n                f'train acc {metric[1] / metric[2]:.3f}')\n    if valid_iter is not None:\n        measures += f', valid acc {valid_acc:.3f}'\n    print(measures + f'\\n{metric[2] * num_epochs / timer.sum():.1f}'\n          f' examples/sec on {str(devices)}')","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.311614Z","iopub.status.idle":"2023-04-09T09:49:05.312456Z","shell.execute_reply.started":"2023-04-09T09:49:05.312180Z","shell.execute_reply":"2023-04-09T09:49:05.312210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"devices, num_epochs, lr, wd = d2l.try_all_gpus(), 20, 2e-4, 5e-4\nlr_period,  net = 4,  get_net()\nlr_decay = 0.9\ntrain(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period,lr_decay)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.313918Z","iopub.status.idle":"2023-04-09T09:49:05.314759Z","shell.execute_reply.started":"2023-04-09T09:49:05.314498Z","shell.execute_reply":"2023-04-09T09:49:05.314523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net, preds = get_net(), []\ntrain(net, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period,\n      lr_decay)\n\nfor X, _ in test_iter:\n    y_hat = net(X.to(devices[0]))\n    preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())\nsorted_ids = list(range(1, len(test_ds) + 1))\nsorted_ids.sort(key=lambda x: str(x))\ndf = pd.DataFrame({'id': sorted_ids, 'label': preds})\ndf['label'] = df['label'].apply(lambda x: train_valid_ds.classes[x])\ndf.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T09:49:05.316250Z","iopub.status.idle":"2023-04-09T09:49:05.317087Z","shell.execute_reply.started":"2023-04-09T09:49:05.316808Z","shell.execute_reply":"2023-04-09T09:49:05.316835Z"},"trusted":true},"execution_count":null,"outputs":[]}]}