{"cells":[{"metadata":{},"cell_type":"markdown","source":"## 使用VGG-16和ResNet对CIFAR-10进行分类，\n## 要求：两种网络自行编写与训练，禁止使用预训练模型。不使用预训练模型\n\ncifar-10数据集：包含train、test文件夹，以及一个trainLabels.csv。train file contain 50000 images, and test file contain 10000 office test images and 290000 junk images。"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from mxnet import autograd, gluon, init, nd\nfrom mxnet.gluon import data as gdata, loss as gloss,nn\nimport mxnet as mx\nimport os\nimport pandas as pd\nimport shutil          \nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#先使用小容量的样本进行训练\ndata_source = '../input/cifar10zip-for-train-and-test'\ntrain_source = 'train/train'\ntest_source = 'test/test'\n\ndemo = False\nif demo:\n    train_dir, test_dir, batch_size = 'train_tiny', 'test_tiny', 1\n    data_dir = '/kaggle/working/cifartiny'\nelse:\n    train_dir, test_dir, batch_size = 'train', 'test', 128\n    data_dir, label_file = '/kaggle/working', 'trainLabels.csv'\ninput_dir, valid_ratio = 'train_valid_test', 0.1\n\n#下面定义一个辅助函数，从而仅在路径不存在的情况下创建路径。\ndef mkdir_if_not_exist(path):  # 函数创建目录\n    if not os.path.exists(os.path.join(*path)):   #此处考虑path可能为列表的形式\n        print(os.path.join(*path))\n        os.makedirs(os.path.join(*path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#在‘/kaggle/working/'里新建cifartiny，用于后面的读取\n#mkdir_if_not_exist([data_dir, train_dir])\nshutil.copytree(os.path.join(data_source, train_source), os.path.join(data_dir, train_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#读取测试集的标签文件，valid_ratio是验证集占原始训练集的比例。\n#读取trainLabels.csv里的序号和label,将它们转为字典类型\ndef read_label_file(data_dir, label_file, train_dir, valid_ratio):\n    with open(os.path.join(data_dir, label_file), 'r') as f:\n        lines = f.readlines()[1:]  # 跳过文件头行（栏名称，id和label）\n        #此时lines为列表类型，其中元素是由序号+ 逗号 +label+\\n组成的字符串\n        tokens = [l.rstrip().split(',') for l in lines]  #rstrip()默认在字符串尾删除\\n、\\t、空格等字符串\n        #tokens亦为列表类型，其中元素由split隔开，为包含两个子元素的子列表类型\n        idx_label = dict(((int(idx), label) for idx, label in tokens))\n        #先加括号构造含有两个字符串的元组，再构造generator、构造字典类型\n    labels = set(idx_label.values())       #构造元组，删除重复元素\n    n_train_valid = len(os.listdir(os.path.join(data_dir, train_dir)))     #统计文件夹train_dir中子文件（图片）的个数\n    n_train = int(n_train_valid * (1 - valid_ratio))      #计算训练数目\n    assert 0 < n_train < n_train_valid        #判断valid_ratio是否有效\n    return n_train // len(labels), idx_label        # //代表整除，返回每个label所需的训练数，以及字典类型的label文件\n\nn_train_per_label, idx_label = read_label_file(data_dir, label_file, train_dir, valid_ratio)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#下面定义一个辅助函数，从而仅在路径不存在的情况下创建路径。\ndef mkdir_if_not_exist(path):  \n    if not os.path.exists(os.path.join(*path)):\n        os.makedirs(os.path.join(*path))\n    \n#切割训练集，以valid_ratio为标准\n#在input_dir下新增文件夹train_valid, train, valid, 其中分别包含原始训练集、训练数据集以及测试数据集\n#三个子文件夹里均是各个类型所包含的图片\ndef reorg_train_valid(data_dir, train_dir, input_dir, n_train_per_label,\n                      idx_label):\n    #input_dir为新建的文件夹,为train_valid_test\n    label_count = {}  #构造空字典\n    for train_file in os.listdir(os.path.join(data_dir, train_dir)):\n        #train_file的一个例子：23.png。\n        idx = int(train_file.split('.')[0])   \n        label = idx_label[idx]  #str类型,与字典序号相对应\n        mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])\n        #构造新文件夹train_valid,里面存在包含各个类型图片的小文件夹\n        shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                    os.path.join(data_dir, input_dir, 'train_valid', label))\n        #将各个png图片放到他们label对应的文件夹里\n        if label not in label_count or label_count[label] < n_train_per_label:\n            #当字典的key进行查找或者每个label的图片数小于预定值\n            mkdir_if_not_exist([data_dir, input_dir, 'train', label])\n            #构造新文件夹train,里面也是包含各个类型图片的小文件夹\n            \n            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                        os.path.join(data_dir, input_dir, 'train', label))\n            label_count[label] = label_count.get(label, 0) + 1\n            #字典可以通过指定key的value来增加dict中的元素，get方法返回指定键的值，如不存在则返回0\n        else:\n            mkdir_if_not_exist([data_dir, input_dir, 'valid', label])\n            #构建新文件夹valid,存放train_valid中包含但是train中不包含的值，即用于验证的值\n            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                        os.path.join(data_dir, input_dir, 'valid', label))\n    \n    #print('train_valid')\n    #for dirpath, dirnames, filenames in os.walk(os.path.join(data_dir, input_dir, 'train_valid')):\n    #    file_count = 0\n    #    for file in filenames:\n    #        file_count = file_count +1\n    #    print(dirpath, file_count)\n    #print(label_count)\nreorg_train_valid(data_dir, train_dir, input_dir, n_train_per_label, idx_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#下面的reorg_test函数用来整理测试集，从而方便预测时的读取。\ndef reorg_test(data_dir, test_dir, input_dir):\n    mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])\n    for test_file in os.listdir(os.path.join(data_dir, test_dir)):\n        shutil.copy(os.path.join(data_dir, test_dir, test_file),\n                    os.path.join(data_dir, input_dir, 'test', 'unknown'))        \nreorg_test(data_dir, test_dir, input_dir) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#进行简易的图像增广\ntransform_train = gdata.vision.transforms.Compose([\n    gdata.vision.transforms.Resize(40),\n    gdata.vision.transforms.RandomResizedCrop(32, scale = (0.64, 1.0), ratio = (1.0, 1.0)),\n    #先裁剪出原图像0.64-1.0倍的图像，在变换为大小为32的图像\n    gdata.vision.transforms.RandomFlipLeftRight(),    #图像左右翻转\n    gdata.vision.transforms.ToTensor(),\n    gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010]) \n])\ntransform_test = gdata.vision.transforms.Compose([\n    gdata.vision.transforms.ToTensor(),\n    gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 从文件夹中构建数据集，从数据集中获取小批量"},{"metadata":{"trusted":true},"cell_type":"code","source":"#加载文件内容构造数据集，flag为1表明输入图像含有3个通道，即彩色\ntrain_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'train'), flag = 1)\nvalid_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'valid'), flag = 1)\ntrain_valid_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'train_valid'), flag = 1)\ntest_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'test'), flag = 1)\n\n#从数据集加载数据并返回小批量数据。\n#train_ds返回带有每个样本的第一个元素作为输入、由函数fn转换的数据集。在保持label不变的同时转换数据\ntrain_iter = gdata.DataLoader(train_ds.transform_first(transform_train), batch_size, shuffle = True, last_batch = 'keep')\n#gdata.vision.ImageFolderDataset.transform_first(fn, lazy)，其中lazy默认为False，转换个例，否则转换所有。当fn为随机时，必须为False\nvalid_iter = gdata.DataLoader(valid_ds.transform_first(transform_test), batch_size, shuffle = True, last_batch = 'keep')\ntrain_valid_iter = gdata.DataLoader(train_valid_ds.transform_first(transform_train), batch_size, shuffle = True, last_batch = 'keep')\ntest_iter = gdata.DataLoader(test_ds.transform_first(transform_test), batch_size, shuffle = True, last_batch = 'keep')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 定义模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"#此处的ResNet与之前的不同：在Residual上使用HybirdBlock而不是Block,定义的顺序网络改为HybirdSequential()\n#首个卷积层使用通道为64，步幅为1的3*3卷积层，其后接BatchNorm和relu函数，而没有最大池化层\n\n#传统的ResNet在residual类中为2个具有相同通道数的3*3卷积层，后接批量归一化层和激活函数。取定strides和use_1conv后，第一个卷积层和self.conv3\n#(即use_1conv)可以对输出形状进行减半。在残差块中对非首块的首层网络取定strides和use_1conv使得对非首块的输出减半，在Resnet中的首个卷积层\n#是通道数为64，步幅大小为2的7*7卷积层，后接批量归一、激活函数，以及步幅为2的3*3最大池化层\nclass Residual(nn.HybridBlock):\n    def __init__(self, num_channels, use_1conv = False, strides = 1, **kwargs):\n        super(Residual, self).__init__(**kwargs)\n        self.conv1 = nn.Conv2D(num_channels, kernel_size = 3, strides = strides, padding = 1)\n        self.conv2 = nn.Conv2D(num_channels, kernel_size = 3, padding = 1)\n        if use_1conv:\n            self.conv3 = nn.Conv2D(num_channels, kernel_size = 1, strides = strides)\n        else:\n            self.conv3 = None\n        self.bn1 = nn.BatchNorm()\n        self.bn2 = nn.BatchNorm()\n    \n    def hybrid_forward(self, F, X):\n        Y = F.relu(self.bn1(self.conv1(X)))\n        Y = self.bn2(self.conv2(Y))\n        if self.conv3:\n            X = self.conv3(X)\n        return F.relu(X + Y)\n            \ndef resnet_18(num_classes):\n    net = nn.HybridSequential()\n    net.add(nn.Conv2D(64, kernel_size = 3, padding = 1), nn.BatchNorm(), nn.Activation('relu'))\n    def resnet_block(num_channels, num_residuals, first_block = False):\n        blk = nn.HybridSequential()\n        for i in range(num_residuals):\n            if i == 0 and not first_block:\n                blk.add(Residual(num_channels, use_1conv = True, strides = 2))\n            else:\n                blk.add(Residual(num_channels))\n        return blk\n    net.add(\n        resnet_block(64, 2, first_block = True),\n        resnet_block(128, 2),\n        resnet_block(256, 2),\n        resnet_block(512, 2)\n    )\n    net.add(nn.GlobalAvgPool2D(), nn.Dense(num_classes))\n    return net\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#构造网络并进行初始化\ndef try_gpu():\n    \"\"\"If GPU is available, return mx.gpu(0); else return mx.cpu().\"\"\"\n    try:\n        ctx = mx.gpu()\n        _ = nd.array([0], ctx=ctx)\n    except mx.base.MXNetError:\n        ctx = mx.cpu()\n    return ctx\n\ndef get_net(ctx):\n    num_classes = 10\n    net = resnet_18(num_classes)\n    net.initialize(ctx = ctx, init = init.Xavier())\n    return net\n\ndef evaluate_accuracy(data_iter, net, ctx=mx.cpu()):\n    \"\"\"Evaluate accuracy of a model on the given data set.\"\"\"\n    \n    acc_sum, n = nd.array([0], ctx = ctx), 0\n    for X, y in data_iter:\n        X = X.as_in_context(ctx)\n        y = y.astype('float32').as_in_context(ctx)\n        acc_sum += (net(X).argmax(axis=1) == y).sum()\n        n += y.size\n\n    return acc_sum.asscalar() / n\n\nloss = gloss.SoftmaxCrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#momentum对初始化的权值进行优化，wd对权重进行弱化，权重越低，复杂度越低，拟合的效果越好\n#gluon.Trainer.step()函数在内调用allreduce_grads()和update()对参数进行更新\n#梯度由1/batch_size进行标准化\ndef train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay):\n    trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate':lr, 'momentum':0.9, 'wd':wd})\n    for epoch in range(num_epochs):\n        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()\n        if epoch > 0 and epoch % lr_period == 0:\n            trainer.set_learning_rate(learning_rate * lr_decay)\n        for X,y in train_iter:\n            y = y.astype('float32').as_in_context(ctx)\n            with autograd.record():\n                y_hat = net(X.as_in_context(ctx))\n                l = loss(y_hat, y).sum()\n            l.backward()\n            train_l_sum += l.asscalar()\n            train_acc_sum += (y_hat.argmax(axis = 1) == y).sum().asscalar()\n            n += y.size\n        time_s = 'time %.2f sec'% (time.time() - start)\n        if valid_iter is not None:\n            valid_acc = evaluate_accuracy(valid_iter, net, ctx)\n            epoch_s = 'epoch %d, loss %f, train_acc %f, valid_acc %f'%(epoch + 1, train_l_sum/n, train_acc_sum/n, valid_acc)\n        else:\n            epoch_s = 'epoch %d, loss %f, train_acc %f' %(epoch +1, train_l_sum/n, train_acc_sum/n)    \n    print(epoch_s + time_s + ' ,lr ' + str(trainer.learning_rate))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 开始训练"},{"metadata":{"trusted":true},"cell_type":"code","source":"ctx, num_epochs, lr, wd = try_gpu(), 1, 0.1, 5e-4\nlr_period, lr_decay, net = 80, 0.1, get_net(ctx)\nnet.hybridize()\ntrain(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 对测试集进行分类并导出csv文件"},{"metadata":{"trusted":true},"cell_type":"code","source":"net, preds = get_net(ctx), []\nnet.hybridize()\ntrain(net, train_valid_iter, None, num_epochs, lr, wd, ctx, lr_period, lr_decay)\nfor X, _ in test_iter:\n    y_hat = net(X.as_in_context(ctx))\n    preds.extend(y_hat.argmax(axis =1).astype('int32').asnumpy())\nsorted_ids = list(range(1, len(test_ds) + 1))\nsorted_ids.sort(key = lambda x: str(x))\ndf = pd.DataFrame({'id': sorted_ids, 'label':preds})\nprint(df)\ndf['label'] = df['label'].apply(lambda x: train_valid_ds.synsets[x])\ndf.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(sorted_ids, preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}