{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Content\n\nUsing PyToch to train model.\n\n1. Analyze MNIST dataset\n    - spectification of dataset\n    - distribution of each class.\n2. Train model (base model) (includes base layers: Conv, Maxpool, Dropout, Relu, FC)\n3. Evaluate result\n    - Plot loss training to track training phase.\n    - show confusion matrix\n    - show misclassified samples\n4. Finetune model by adding BatchNorm layer.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nimport time\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nif torch.cuda.is_available():\n    use_cuda = True\nelse:\n    use_cuda = False\n    \nclass MNISTDataset(Dataset):\n    def __init__(self, csv_file):\n        csv = pd.read_csv(csv_file) \n        self.data = csv.drop('label', axis=1).values\n        self.label = csv['label'].values\n        del csv\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        img, mask = self.data[idx].reshape(1,28,28), self.label[idx]\n        img, mask = np.asarray(img), np.asarray(mask)\n        img, mask = torch.from_numpy(img.astype(np.float32)), torch.from_numpy(mask.astype(np.float32))\n        img = img/255\n        return img, mask","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# analyze train set\ntrain_set = MNISTDataset('/kaggle/input/digit-recognizer/train.csv')\n\ntrain_size = len(train_set)\nsample_idx = np.random.randint(0, len(train_set), dtype=np.int64)\nimdata, imlabel = train_set[sample_idx]\nprint('Số lượng mẫu trong tập huấn luyện gốc: ', train_size)\nprint('Mẫu tại chỉ số: ', sample_idx) #Lấy ngẫu nhiên\nprint(' + Kiểu dữ liệu của mẫu: ', type(imdata))\nprint(' + Kiểu dữ liệu của nhãn: ', type(imlabel))\n\nimdata = np.array(imdata)\nprint(' + Kích thước một mẫu dữ liệu: ', imdata.shape)\nprint(' + Giá trị nhỏ nhất trong array: ', imdata.min())\nprint(' + Giá trị lớn nhất trong array: ', imdata.max())\nprint(' + Giá trị trung bình trong array: ', imdata.mean())\nprint(' + Giá trị std trong array: ', imdata.std())\nimdata = imdata.reshape(28,28)\n\nplt.figure(figsize=(2,2))\nplt.imshow(imdata,  cmap='gray')\nplt.axis('off')\ntitle = 'Nhãn: {:d}'.format(int(imlabel))\nplt.title(title)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\nfrom matplotlib.colors import NoNorm\n\ndef show_random_image(dataset, nrows, ncols):\n    _, axarr = plt.subplots(nrows, ncols, figsize=(10, 6))\n    for r in range(nrows):\n      for c in range(ncols):\n        sample_idx = np.random.randint(0, len(dataset), dtype=np.int64)\n        imdata, imlabel = dataset[sample_idx]\n        imdata = imdata.numpy()\n        imlabel = int(imlabel)\n        imdata = imdata.reshape((28,28))\n     \n        axarr[r,c].imshow(imdata,  cmap='gray',norm=NoNorm())\n        axarr[r,c].set_title(imlabel)\n        plt.setp(axarr[r,c].get_xticklabels(), visible=False)\n        plt.setp(axarr[r,c].get_yticklabels(), visible=False)\n        axarr[r,c].grid(False)\n    plt.show()\n\nshow_random_image(train_set, 5, 10) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib as mplt\n\ndef plot_dataset_stats(datasets, legends, class_names=None):\n    # Thống kê số lượng dữ liệu các nhãn trong các tập dữ liệu\n    colors = ['red', 'green', 'blue', 'maganta', 'cyan']\n    num_classes = 0\n    pos = []\n    max_count = 0\n    for idx in range(len(datasets)):\n        cur_dataset = datasets[idx]\n        cur_legend = legends[idx]\n        labels = [cur_dataset[sid][1] for sid in range(len(cur_dataset))]\n        label_hist = np.bincount(labels)\n        num_classes = max(num_classes, len(label_hist))\n\n        pos = 5.0*np.arange(num_classes) + idx*1.0\n        plt.bar(pos, label_hist, align='center', \n              alpha=0.5, color=colors[idx], label=cur_legend)\n        max_count = max(max_count, label_hist.max())\n    if class_names is None:\n        class_names = [str(idx) for idx in range(num_classes)]\n\n    plt.xticks(pos, class_names)\n    plt.ylabel('The number of samples')\n    plt.xlabel('Label')\n    plt.ylim(0, max_count + 500)\n    plt.title('The number sample of each labels')\n    plt.legend()\n    plt.show()\n\ntrain_dataset = MNISTDataset('/kaggle/input/digit-recognizer/train.csv')\n\n# split train test => train_set and val_set\nTRAIN_SIZE_PRC = 0.7 # train (70%) và val (30%)\n\nTRAIN_SIZE = int(TRAIN_SIZE_PRC*len(train_dataset))\nVALID_SIZE = len(train_dataset) - TRAIN_SIZE\ntrain_dataset, valid_dataset = torch.utils.data.random_split(train_dataset,(TRAIN_SIZE, VALID_SIZE))\n\nplot_dataset_stats([train_dataset, valid_dataset],\n                  ['Train_set', 'Val_set'])\n\nprint('Train set - size: ', TRAIN_SIZE)\nlabels = [train_dataset[sid][1] for sid in range(len(train_dataset))]\nprint('The number sample of each labels (0->9): ', np.bincount(labels))\nprint('Val set - size: ', VALID_SIZE)\nlabels = [valid_dataset[sid][1] for sid in range(len(valid_dataset))]\nprint('The number sample of each labels (0->9): ', np.bincount(labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create data loader\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset,\n    batch_size=128,\n    shuffle=True, \n    num_workers=4, \n    pin_memory=True) \nvalid_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=128, \n    shuffle=False, \n    num_workers=4, \n    pin_memory=True)\n\ndef show_info(ds_name, dataset, loader):\n    print('Tên tập dữ liệu:', ds_name)\n    print('Số lượng mẫu trong tập:', len(dataset))\n    print('Số lượng batch được chia:', len(loader))\n    print('Thông tin của từng batch:')\n\n    for batch_idx, (batch_data, batch_label) in enumerate(loader):\n        IDX = '{:5d}'.format(batch_idx + 1)\n        print(IDX, \"input: \", batch_data.shape,\"target: \", batch_label.shape)\n        break\n\nprint('-'*80)\nshow_info('TẬP HUẤN LUYỆN', train_dataset, train_loader)\nprint('-'*80)\nshow_info('TẬP KIỂM THỬ', valid_dataset, valid_loader)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils functions\n\n* define utils function using in training phase\n1. train: process (forward, update, compute metric) in one epoch\n2. validate: compute metric in val set\n3. AverageMeter: to update score metric (average ,..)\n5. adjust_learning_rate: learining rate scheduler\n6. metric: compute metric evaluation (precision)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\nBEST_MODEL_FNAME = './best_model.data'\nbest_acc = 0\n\ndef train(train_loader, model, criterion, optimizer, epoch, use_cuda=False, print_freq=40):\n    \"\"\"\n        Run one train epoch\n    \"\"\"\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    top1 = AverageMeter()\n\n    # switch to train mode\n    model.train()\n\n    end = time.time()\n    for i, (data, target) in enumerate(train_loader):\n        if use_cuda:\n            data = data.cuda()\n            target = target.long().cuda()\n        else:\n            data = data\n            target = target.long()\n        \n        scores = model(data)\n        loss = criterion(scores, target)\n        \n        # compute gradient and do SGD step\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        scores = scores.float()\n        scores = F.softmax(scores, dim=1)\n        loss = loss.float()\n        \n        # measure precision and record loss\n        prec1 = metric(scores.data, target.data)[0]\n        losses.update(loss.item(), data.size(0))\n        top1.update(prec1.item(), data.size(0))\n\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n\n        if i % print_freq == 0:\n            print('Epoch: [{0}][{1}/{2}]\\t'\n                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                  'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(\n                      epoch, i, len(train_loader), batch_time=batch_time, loss=losses, top1=top1))\n    \n    return losses.avg, top1.avg\n\ndef validate(val_loader, model, criterion, use_cuda=True, print_freq=40):\n    \"\"\"\n    Run evaluation\n    \"\"\"\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    top1 = AverageMeter()\n    global best_acc\n\n    # switch to evaluate mode\n    model.eval()\n\n    end = time.time()\n    for i, (data, target) in enumerate(val_loader):\n        if use_cuda:\n            target = target.cuda()\n            input_var = data.cuda()\n            target_var = target.long().cuda()\n        else:\n            input_var = data\n            target_var = target.long()\n        \n        # compute output\n        scores = model(input_var)\n        loss = criterion(scores, target_var)\n\n        scores = scores.float()\n        scores = F.softmax(scores, dim=1)\n        loss = loss.float()\n\n        prec1 = metric(scores.data, target_var.data)[0]\n        losses.update(loss.item(), data.size(0))\n        top1.update(prec1.item(), data.size(0))\n\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n\n        if i % print_freq == 0:\n            print('Test: [{0}/{1}]\\t'\n                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                  'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(\n                      i, len(val_loader), batch_time=batch_time, loss=losses,\n                      top1=top1))\n          \n    print(' * Prec@1 {top1.avg:.3f}'.format(top1=top1))\n    \n    #save\n    if top1.avg > best_acc:\n          best_acc = top1.avg\n          torch.save(model.state_dict(), BEST_MODEL_FNAME)\n          print('Mô hình tốt hơn đã được tìm thấy => lưu lại')\n\n    return losses.avg, top1.avg, best_acc\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef adjust_learning_rate(optimizer, epoch):\n    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 30 epochs\"\"\"\n    lr = base_lr * (0.5 ** (epoch // 30))\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = lr\n\ndef metric(output, target, topk=(1,)):\n    \"\"\"Computes the precision@k for the specified values of k\"\"\"\n    maxk = max(topk)\n    batch_size = target.size(0)\n\n    _, pred = output.topk(maxk, 1, True, True)\n    pred = pred.t()\n    correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n    res = []\n    for k in topk:\n        correct_k = correct[:k].view(-1).float().sum(0)\n        res.append(correct_k.mul_(100.0 / batch_size))\n    return res","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base model\n\n(conv -> pool -> relu)^2 -> drop -> FC -> relu -> drop -> FC"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n        self.pool1 = nn.MaxPool2d(kernel_size=2)\n        self.relu1 = nn.ReLU()\n        \n        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n        self.pool2 = nn.MaxPool2d(kernel_size=2)\n        self.relu2 = nn.ReLU()\n        \n        self.conv2_drop = nn.Dropout2d(p=0.5)\n        \n        self.fc1 = nn.Linear(320, 50)\n        self.fc1_relu = nn.ReLU()\n        self.fc1_drop = nn.Dropout2d(p=0.5)\n        self.fc2 = nn.Linear(50, 10)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pool1(x)\n        x = self.relu1(x)\n        \n        x = self.conv2(x)\n        x = self.conv2_drop(x)\n        x = self.pool2(x)\n        x = self.relu2(x)\n        \n        x = x.view(-1, 320)\n        \n        x = self.fc1(x)\n        x = self.fc1_relu(x)\n        x = self.fc1_drop(x)\n        x = self.fc2(x)\n        return x\n  \nmodel = Net()\nprint(model)\n\nbase_lr = 0.05\nmomentum = 0.9\nweight_decay = 5e-4\nnepochs = 120\n\nloss_func = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(),\n                            base_lr,\n                            momentum=momentum,\n                            weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_lst = {}\nloss_lst['train'] = []\nloss_lst['val']   = []\n\nprec_lst = {}\nprec_lst['train'] = []\nprec_lst['val']   = []\n\nfor epoch in range(0, nepochs):\n    adjust_learning_rate(optimizer, epoch)\n    if use_cuda:\n        model.cuda()\n        loss_func.cuda()\n        loss_train, prec_train = train(train_loader, model, loss_func, optimizer, epoch, True, print_freq=70)\n        loss_val, prec_val, best_avg = validate(valid_loader, model, loss_func, True, print_freq=50)\n        \n        loss_lst['train'].append(loss_train)\n        loss_lst['val'].append(loss_val)\n        prec_lst['train'].append(prec_train)\n        prec_lst['val'].append(prec_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Display training loss by epoch"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1)\nax[0].plot(loss_lst['train'], color='b', label=\"Training loss\")\nax[0].plot(loss_lst['val'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(prec_lst['train'], color='b', label=\"Training precision\")\nax[1].plot(prec_lst['val'], color='r',label=\"Validation precision\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport itertools\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix of val set',\n                          cmap=plt.cm.Blues):\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_val = np.array([])\npred_val = np.array([])\ngt_val = np.array([])\n\nprint(\"Best precision of val phase:\", best_avg)\nfor i, (data, target) in enumerate(valid_loader):\n    target = target.numpy().astype(np.int)\n    pred = model(data.cuda())\n    pred = pred.float()\n    pred = F.softmax(pred, dim=1)\n    pred = torch.argmax(pred,1).cpu().numpy()\n    \n    x_val = np.concatenate((x_val, data.numpy()), axis=0) if x_val.shape[0] != 0 else data.numpy()\n    pred_val = np.append(pred_val, pred).astype(np.int)\n    gt_val = np.append(gt_val, target).astype(np.int)\n    \nconfusion_mtx = confusion_matrix(gt_val, pred_val) \nplot_confusion_matrix(confusion_mtx, classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display some misclassified samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"errors = (pred_val - gt_val != 0)\nx_val_err = x_val[errors]\ny_pred_err = pred_val[errors]\ngt_err = gt_val[errors]\n\ndef display_errors(img_errors, pred_errors, gt_errors):\n    n = 0\n    nrows = 3\n    ncols = 3\n    fig, ax = plt.subplots(nrows, ncols, figsize=(10,10))\n    for row in range(nrows):\n        for col in range(ncols):\n            index = np.random.randint(0, gt_errors.shape[0], dtype=np.int64)\n            ax[row,col].imshow((img_errors[index]).reshape((28,28)), cmap='gray')\n            ax[row,col].set_title(\"Pred label :{}\\nTrue label :{}\".format(pred_errors[index], gt_errors[index]))\n            n += 1\n            \n    fig.tight_layout()\n    plt.show()\n\ndisplay_errors(x_val_err, y_pred_err, gt_err)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Finetune Model -- add BatchNorm"},{"metadata":{"trusted":true},"cell_type":"code","source":"class NetBN(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n        self.bn1 = nn.BatchNorm2d(10)\n        self.pool1 = nn.MaxPool2d(kernel_size=2)\n        self.relu1 = nn.ReLU()\n        \n        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n        self.bn2 = nn.BatchNorm2d(20)\n        self.pool2 = nn.MaxPool2d(kernel_size=2)\n        self.relu2 = nn.ReLU()\n        \n        self.drop = nn.Dropout2d(p=0.5)\n        \n        self.fc1 = nn.Linear(320, 50)\n        self.fc1_relu = nn.ReLU()\n        self.fc1_drop = nn.Dropout2d(p=0.5)\n        self.fc2 = nn.Linear(50, 10)\n\n    def forward(self, x):\n        x = self.bn1(self.conv1(x))\n        x = self.pool1(x)\n        x = self.relu1(x)\n        \n        x = self.bn2(self.conv2(x))\n        x = self.drop(x)\n        x = self.pool2(x)\n        x = self.relu2(x)\n        \n        x = x.view(-1, 320)\n        \n        x = self.fc1(x)\n        x = self.fc1_relu(x)\n        x = self.fc1_drop(x)\n        x = self.fc2(x)\n        return x\n  \nmodel = NetBN()\nprint(model)\n\nbase_lr = 0.05\nmomentum = 0.9\nweight_decay = 5e-4\nnepochs = 120\n\nloss_func = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(),\n                            base_lr,\n                            momentum=momentum,\n                            weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_lst = {}\nloss_lst['train'] = []\nloss_lst['val']   = []\n\nprec_lst = {}\nprec_lst['train'] = []\nprec_lst['val']   = []\n\nfor epoch in range(0, nepochs):\n    adjust_learning_rate(optimizer, epoch)\n    if use_cuda:\n        model.cuda()\n        loss_func.cuda()\n        loss_train, prec_train = train(train_loader, model, loss_func, optimizer, epoch, True, print_freq = 50)\n        loss_val, prec_val, best_avg = validate(valid_loader, model, loss_func, True, print_freq = 50)\n        \n        loss_lst['train'].append(loss_train)\n        loss_lst['val'].append(loss_val)\n        prec_lst['train'].append(prec_train)\n        prec_lst['val'].append(prec_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_val = np.array([])\npred_val = np.array([])\ngt_val = np.array([])\n\nprint(\"Best precision of val phase:\", best_avg)\nfor i, (data, target) in enumerate(valid_loader):\n    target = target.numpy().astype(np.int)\n    pred = model(data.cuda())\n    pred = pred.float()\n    pred = F.softmax(pred, dim=1)\n    pred = torch.argmax(pred,1).cpu().numpy()\n    \n    x_val = np.concatenate((x_val, data.numpy()), axis=0) if x_val.shape[0] != 0 else data.numpy()\n    pred_val = np.append(pred_val, pred).astype(np.int)\n    gt_val = np.append(gt_val, target).astype(np.int)\n    \nconfusion_mtx = confusion_matrix(gt_val, pred_val) \nplot_confusion_matrix(confusion_mtx, classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Summary\n|                      \t|   Val  | Test   |\n|-----------------------| ------ | ------ |\n|       Base Model     \t| 99.161 | 98.653 |\n|  Finetune 1 - add BN \t| 99.001 | 98.039 |\n\n\nHiện tại, với bài toán nhận diện chữ số thì chỉ cần mô hình neuron đơn giản bao gồm các lớp Conv, Relu, Dropout, FC thì bài toán này cũng đã đạt được kết quả khả quan cao trên 98%. \n\nDo đó với những tiêu chuẩn hiện tại để so sánh các giải thuật / mô hình thì chưa thể đánh giá tốt được liệu giải thuật đó có hiệu quả hay không bởi sự hạn chế của dữ liệu. Dữ liệu hiện tại chỉ tập trung vào những ảnh có môi trường giống nhau -> ví dụ khi ra thực tế thì có thể những con số có màu đỏ nền trắng hoặc ngược lại thay vì sự phân bố trong MNIST, nên việc độ chính xác thu được khá cao cũng chưa nói lên được liệu mô hình này có chạy hay không khi đem áp dụng cho những context khác nhau. \n\nPhương pháp đánh giá, so sánh giữa các thuật toán: \n - Để kiểm tra thuật toán nào phù hợp hơn sẽ phải chọn ra 1 tiêu chí đánh giá chung (single metric evaluation) ví dụ như bài toán ở trên người ta thường chọn độ đo precision để so sánh giữa các bài toán. \n - Độ đo này sẽ được đánh giá trên cùng 1 tập dữ liệu (test set) với tiêu chí tập dữ liệu này sẽ không tham gia vào quá trình huấn luyện của giải thuật.\n \nVà việc chỉ cần mô hình này đã đạt được độ chính xác đủ cao trên dữ liệu này thì việc tìm một thuật toán khác đạt tương đương như vậy cũng không đem lại ý nghĩa."},{"metadata":{},"cell_type":"markdown","source":"# TEST PHASE"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load test set\ntest_dataset = pd.read_csv('/kaggle/input/digit-recognizer/test.csv').values\n\n# prediction\ntest_set = torch.from_numpy(test_dataset.reshape(-1, 1, 28, 28)).to(torch.float32)\npred = model(test_set.cuda())\npred = pred.float()\npred = F.softmax(pred, dim=1)\npred = torch.argmax(pred,1).cpu().numpy()\n\n# save result to submit\ndf = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\ndf['Label'] = pred\ndf.to_csv('submission.csv',line_terminator='\\r\\n', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}