{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # for plotting beautiful graphs\n\n# train test split from sklearn\nfrom sklearn.model_selection import train_test_split\n\n# Import Torch \nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms, models\n# from torch.utils.data import SubsetRandomSampler\nfrom torch.autograd import Variable\nfrom torch import nn, optim\nimport torch.nn.functional as F\n\n# What's in the current directory?\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", dtype=np.float32)\nfinal_test = pd.read_csv(\"../input/test.csv\", dtype=np.float32)\nsample_sub = pd.read_csv(\"../input/sample_submission.csv\")\ntrain.label.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Seperate the features and labels\ntargets_np = train.label.values\nfeatures_np = train.loc[:, train.columns != 'label'].values/255\n\n# Split into training and test set\nfeatures_train, features_test, target_train, target_test = train_test_split(features_np, targets_np, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable\nfeaturesTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(target_train).type(torch.LongTensor) # data type is long\n\n# create feature and targets tensor for test set.\nfeaturesTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(target_test).type(torch.LongTensor) # data type is long","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 256\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)\ntest = torch.utils.data.TensorDataset(featuresTest,targetsTest)\n\n# data loader\ntrain_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = True)\ntest_loader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 量化和微调\nimport numpy as np\nimport math\nimport os\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nfrom torch.autograd import Function\n\n# 量化工具 量化权重\ndef quantization(weights, bw, fl):\n    v_max = (pow(2, bw-1)-1)*pow(2, -fl)\n    v_min = -1*pow(2, bw-1)*pow(2, -fl)\n    max_out_of_range = (weights>v_max).float()\n    min_out_of_range = (weights<v_min).float()\n    weights *= 1-max_out_of_range\n    weights *= 1-min_out_of_range\n    weights += max_out_of_range*v_max\n    weights += min_out_of_range*v_min\n    weights = torch.round(weights/pow(2, -fl))\n    return weights*pow(2, -fl)\n\n# 自定义量化操作\n\nclass quantization_F(Function):\n    @staticmethod\n    def forward(ctx, input, bw, fl):\n        v_max = (pow(2, bw-1)-1)*pow(2, -fl)\n        v_min = -1*pow(2, bw-1)*pow(2, -fl)\n        max_out_of_range = (input>v_max).float()\n        min_out_of_range = (input<v_min).float()\n        input *= 1-max_out_of_range\n        input *= 1-min_out_of_range\n        input += max_out_of_range*v_max\n        input += min_out_of_range*v_min\n        input = torch.round(input/pow(2, -fl))\n        return input*pow(2, -fl)\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        return grad_output, None, None\n\n# 记录数据流的字典\nfeature_maps = {}\n\n# 目标网络 eval_flag 在需要获取数据流时，需设为1\nclass LeNet(nn.Module):\n    def __init__(self):\n        super(LeNet, self).__init__()\n        self.conv1 = nn.Sequential(     #input_size=(1*28*28)\n            nn.Conv2d(1, 6, 5, 1, 2), #padding=2保证输入输出尺寸相同\n            nn.ReLU(),      #input_size=(6*28*28)\n            nn.MaxPool2d(kernel_size=2, stride=2),#output_size=(6*14*14)\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(6, 16, 5),\n            nn.ReLU(),      #input_size=(16*10*10)\n            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)\n        )\n        self.fc1 = nn.Sequential(\n            nn.Linear(16 * 5 * 5, 120),\n            nn.ReLU()\n        )\n        self.fc2 = nn.Sequential(\n            nn.Linear(120, 84),\n            nn.ReLU()\n        )\n        self.fc3 = nn.Linear(84, 10)\n    # 定义前向传播过程，输入为x\n    def forward(self, x, eval_flag = False):\n        if eval_flag:\n            feature_maps['input'] = x\n            x = self.conv1(x)\n            feature_maps['conv1'] = x\n            x = self.conv2(x)\n            feature_maps['conv2'] = x\n            x = x.view(x.size()[0], -1)\n            x = self.fc1(x)\n            feature_maps['fc1'] = x\n            x = self.fc2(x)\n            feature_maps['fc2'] = x\n            x = self.fc3(x)\n            feature_maps['fc3'] = x\n        else:\n            x = self.conv1(x)\n            x = self.conv2(x)\n            x = x.view(x.size()[0], -1)\n            x = self.fc1(x)\n            x = self.fc2(x)\n            x = self.fc3(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate our model\nmodel = LeNet().cuda()\n# Define our loss function\nerror = nn.CrossEntropyLoss()\n# Define the optimier\nlearning_rate = 0.01\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\ncount = 0\n\nepochs = 25\nsteps = 0\nprint_every = 50\nloss_list = []\niteration_list = []\naccuracy_list = []\nnum_epochs  = 20\nfor epoch in range(20):\n    if epoch==10:\n        for param_group in optimizer.param_groups:\n            param_group['lr'] *= 0.1\n    for i, (images, labels) in enumerate(train_loader):   \n        train = Variable(images.view(-1,1,28,28)).cuda()\n        labels = Variable(labels).cuda()\n        optimizer.zero_grad()\n        outputs = model(train)\n        loss = error(outputs, labels)\n        loss.backward()\n\n        optimizer.step()\n        count += 1\n        if count % 50 == 0:\n            correct = 0\n            total = 0\n            for images, labels in test_loader:\n                test = Variable(images.view(-1,1,28,28)).cuda()    \n                outputs = model(test)\n                predicted = torch.max(outputs.data, 1)[1]\n                total += len(labels)\n                correct += (predicted.cpu() == labels).sum()\n            accuracy = 100 * correct / float(total)\n            loss_list.append(loss.data)\n            iteration_list.append(count)\n            accuracy_list.append(accuracy)\n            if count % 500 == 0:\n                print('Iteration: {}  Loss: {}  Accuracy: {} %'.format(count, loss.item(), accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 统计权重的最大值\nd = model.state_dict()\nstored_w = {}\nfor k,w in d.items():\n    w = w.view(1,-1).abs()\n    stored_w[k] = [(w[0,:]).max(), (w[0,:]).min()]\nstored_w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 计算权重的位数\nint_w = {}\nfor k,w in stored_w.items():\n    int_w[k] = int(1+ np.ceil(np.log2(w[0].cpu())))\nint_w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 计算量化后的权重\nq_state_dict = {}\nbit_width = 8\nfor k,w in d.items():\n    q_state_dict[k] = quantization(w, bit_width, bit_width-int_w[k])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 计数据流中最大的数值\nmodel.eval().cuda()\nblob_data = {}\nfor images, labels in test_loader:\n    test = Variable(images.view(-1,1,28,28)).cuda()\n    outputs = model(test, eval_flag = True)\n    for k in feature_maps.keys():\n        tmp_max = feature_maps[k].max().cpu().data.numpy()\n        if k in blob_data:\n            if tmp_max>blob_data[k]:\n                blob_data[k] = tmp_max\n        else:\n            blob_data[k] = tmp_max\n    feature_maps.clear()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 计算数据流的位数\nint_blob = {}\nfor k,b in blob_data.items():\n    print(b)\n    int_blob[k] = np.ceil(np.log2(b))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 重建网络，添加量化操作\nclass LeNet_quantization(nn.Module):\n    def __init__(self):\n        super(LeNet_quantization, self).__init__()\n        self.conv1 = nn.Sequential(     #input_size=(1*28*28)\n            nn.Conv2d(1, 6, 5, 1, 2), #padding=2保证输入输出尺寸相同\n            nn.ReLU(),      #input_size=(6*28*28)\n            nn.MaxPool2d(kernel_size=2, stride=2),#output_size=(6*14*14)\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(6, 16, 5),\n            nn.ReLU(),      #input_size=(16*10*10)\n            nn.MaxPool2d(2, 2)  #output_size=(16*5*5)\n        )\n        self.fc1 = nn.Sequential(\n            nn.Linear(16 * 5 * 5, 120),\n            nn.ReLU()\n        )\n        self.fc2 = nn.Sequential(\n            nn.Linear(120, 84),\n            nn.ReLU()\n        )\n        self.fc3 = nn.Linear(84, 10)\n\n    # 定义前向传播过程，输入为x\n    def forward(self, x):\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['input'])\n        x = self.conv1(x)\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['conv1'])\n        x = self.conv2(x)\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['conv2'])\n        x = x.view(x.size()[0], -1)\n        x = self.fc1(x)\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['fc1'])\n        x = self.fc2(x)\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['fc2'])\n        x = self.fc3(x)\n        x = quantization_F.apply(x, bit_width, bit_width-int_blob['fc3'])\n        return x\n\n# 重建网络加载预训练参数\nmodel2 = LeNet_quantization().cuda()\nmodel2.load_state_dict(q_state_dict)\nmodel2.eval()\ncorrect = 0\n\n# 测试量化后的网络\n\nfor images, labels in test_loader:\n    test = Variable(images.view(-1,1,28,28)).cuda()\n    # Forward propagation\n    outputs = model2(test)\n    # Get predictions from the maximum value\n    predicted = torch.max(outputs.data, 1)[1]\n    # Total number of labels\n    total += len(labels)\n    correct += (predicted.cpu() == labels).sum()\naccuracy = 100 * correct / float(total)\nprint(\"Test Accuracy after quantization: {:.3f}%\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 微调量化网络\nerror = nn.CrossEntropyLoss()\nlearning_rate = 0.001\noptimizer = torch.optim.SGD(model2.parameters(), lr=learning_rate)\nmodel2.cuda()\ncount = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_list = []\niteration_list = []\naccuracy_list = []\nnum_epochs  = 10\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):      \n        train = Variable(images.view(-1,1,28,28)).cuda()\n        labels = Variable(labels).cuda()\n        optimizer.zero_grad()       \n        outputs = model2(train)\n        loss = error(outputs, labels)\n        model2.load_state_dict(d)\n        loss.backward()\n        optimizer.step()\n        count += 1\n        # get model2 full width state ==> d ==> quantization == > q_state_dict ===> load q_state_dict\n        d = model2.state_dict()\n        q_state_dict = {}\n        bit_width = 8\n        for k,w in d.items():\n            q_state_dict[k] = quantization(w, bit_width, bit_width-int_w[k])\n        model2.load_state_dict(q_state_dict)\n        \n        # 测试      \n        if count % 50 == 0:\n            correct = 0\n            total = 0\n            for images, labels in test_loader:\n                test = Variable(images.view(-1,1,28,28)).cuda()    \n                outputs = model2(test)\n                predicted = torch.max(outputs.data, 1)[1]\n                total += len(labels)\n                correct += (predicted.cpu() == labels).sum()\n            accuracy = 100 * correct / float(total)\n            loss_list.append(loss.data)\n            iteration_list.append(count)\n            accuracy_list.append(accuracy)\n            if count % 500 == 0:\n                print('Iteration: {}  Loss: {}  Accuracy: {} %'.format(count, loss.item(), accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}