{"cells":[{"metadata":{},"cell_type":"markdown","source":"**List File Path**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use os.listdir to check file path/name\nimport os\npath = '/kaggle/input'\nprint(os.listdir(path))\n\nfilepath = os.path.join(path,'digit-recognizer')\nprint(os.listdir(filepath))\n\ndel path","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Import Train Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\n\n# func to define file path name\ndef path_(filepath, train=None):\n    if train:\n        return os.path.join(filepath, 'train.csv')\n    else:\n        return os.path.join(filepath, 'test.csv')\n\n#import csv file to pandas dataframe\ntrainfile = pd.read_csv(path_(filepath, train=True))\ntestfile = pd.read_csv(path_(filepath))\n\ndel filepath","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Check data shape**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('trainfile shape: {}'.format(trainfile.shape))\nprint('testfile shape: {}'.format(testfile.shape))\nprint(trainfile)\nprint(testfile)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Fix Random Seed**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport torch\nimport matplotlib.pyplot as plt\n\n# fix random seed\nnp.random.seed(0)\n# fix random seed\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\ntorch.cuda.manual_seed_all(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Transfer data to numpy format**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train data null count: {}'.format(trainfile.isnull().sum(axis=1).sum(axis=0)))\n\n# train data to numpy for sklearn function format\n\ntrain_X = trainfile.drop(labels=\"label\", axis=1).to_numpy()\nprint('train data shape: {}'.format(train_X.shape))\n\ntest_X = testfile.to_numpy()\nprint('test  data shape: {}'.format(test_X.shape))\n\n# pandas need to call column by its name\ntrain_Y = trainfile['label'].to_numpy()\nprint('train label shape: {}'.format(train_Y.shape))\n\n# func to count 0 to 9 for train set and valid set\ndef cnt_0to9_(data):\n    cnt = np.zeros((10,2), dtype=int)\n    for i in data:\n        cnt[i-1][0] = i\n        cnt[i-1][1] += 1\n    return cnt\n\n# each label count\naxis = cnt_0to9_(train_Y)\n\nplt.bar(axis[:,0],axis[:,1])\nplt.axis([-1,10, 0,5000])\nplt.xticks(axis[:,0])\nplt.title('all train data')\nplt.ylabel('data count')\nplt.xlabel('number')\nplt.figure(figsize=(12,3))\nplt.show()\nplt.close()\n\ndel axis, trainfile, testfile","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preprocess train & test data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = (train_X/ 255.)\ntest_X = (test_X/ 255.)\n\nmean = train_X.mean()\nstd = train_X.std()\n\ntrain_X = (train_X - mean)/ std\ntest_X = (test_X - mean)/ std\n\ndel mean, std","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**define train set & valid set**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# random split 1:9 (valid:train)\ntrain_x, valid_x, train_y, valid_y = train_test_split(train_X, train_Y, test_size=0.1, shuffle=True)\n\n# check split data uniformity\nsplit_train = cnt_0to9_(train_y)\nsplit_valid = cnt_0to9_(valid_y)\n\ndef bar_plot_(train, valid):\n    fig, (ax1, ax2) = plt.subplots(1, 2)\n    fig.set_figheight(3)\n    fig.set_figwidth(12)\n    ax1.bar(train[:,0], train[:,1])\n    ax1.set_title('train label')\n    ax1.set_xlabel('data count')\n    ax1.set_ylabel('number')\n    ax1.set_xticks(train[:,0])\n    ax2.bar(valid[:,0], valid[:,1])\n    ax2.set_title('valid label')\n    ax2.set_xlabel('data count')\n    ax2.set_ylabel('number')\n    ax2.set_xticks(valid[:,0])\n    plt.show()\n    plt.close()\n\nbar_plot_(split_train, split_valid)\n#bar_plot_(split_valid, 'valid')\n\ndel split_train, split_valid, train_X, train_Y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define dataset class**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nclass mnist_dataset(Dataset):\n    def __init__(self, data, label = None, transform = False):\n        # torch shape (N,C,H,W)\n        self.data = torch.from_numpy(data.astype(np.float32)).reshape(-1,1,28,28)\n        # label exist\n        if label is not None:\n            self.label = torch.from_numpy(label)\n        else:\n            self.label = label\n        # define transformation        \n        train_transforms = transforms.Compose(\n                    [transforms.ToPILImage(),\n                     # MNIST handwritten do not need flip graph\n                     transforms.RandomAffine(degrees=15, translate=(0,0.2), scale=(0.9, 1)), \n                     #transforms.RandomRotation(15),\n                     transforms.ToTensor(),])\n        if transform:\n            self.transforms = train_transforms\n        else:\n            self.transforms = None\n    def __len__(self):\n        return self.data.size(0)\n    def __getitem__(self, idx):\n        x = self.data[idx]\n        if self.label is not None:\n            if self.transforms is not None:\n                x = self.transforms(x)\n            y = self.label[idx]\n            return x, y\n        else:\n            return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define model (refer to LeNet5)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\n\nclass LeNet_(nn.Module):\n    def __init__(self):\n        super(LeNet_,self).__init__()\n        \n        cnn_list = [nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=2),\n                nn.Tanh(),\n                nn.AvgPool2d(2,2),\n                nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0),\n                nn.Tanh(),\n                nn.AvgPool2d(2,2),\n                nn.Conv2d(16, 120, kernel_size=5, stride=1, padding=0),\n                nn.Tanh(),                \n               ]\n        self.cnn = nn.Sequential(*cnn_list)\n    \n        fc_list = [nn.Linear(120, 84),\n               nn.Tanh(),\n               nn.Linear(84,10),\n              ]\n        self.fc = nn.Sequential(*fc_list)\n        \n    def forward(self, x):\n        x = self.cnn(x)\n        x = x.view(x.size(0),-1)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\n\nclass LeNet_mdf(nn.Module):\n    def __init__(self):\n        super(LeNet_mdf, self).__init__()\n        \n        cnn_list = [nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2),                   \n                    nn.ReLU(),\n                    nn.AvgPool2d(2,2),\n                    nn.Conv2d(32, 84, kernel_size=5, stride=1, padding=0),                    \n                    nn.ReLU(),\n                    nn.AvgPool2d(2,2),\n                    nn.Conv2d(84, 169, kernel_size=5, stride=1, padding=0),                    \n                    nn.ReLU(),                \n                   ]\n        self.cnn = nn.Sequential(*cnn_list)\n    \n        fc_list = [nn.Linear(169, 84),\n                   nn.BatchNorm1d(84),\n                   nn.ReLU(),\n                   nn.Dropout(p=0.05),\n                   nn.Linear(84,10),\n                  ]\n        self.fc = nn.Sequential(*fc_list)\n        \n    def forward(self, x):\n        x = self.cnn(x)\n        x = x.view(x.size(0),-1)\n        x = self.fc(x)\n        return x       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\n\nclass LeNet_mdf2(nn.Module):\n    def __init__(self):\n        super(LeNet_mdf2, self).__init__()\n\n        cnn_list = [nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2),                    \n                    nn.ReLU(),\n                    nn.AvgPool2d(2,2),\n                    nn.Conv2d(32, 84, kernel_size=5, stride=1, padding=0),                    \n                    nn.ReLU(),\n                    nn.AvgPool2d(2,2),\n                    nn.Conv2d(84, 120, kernel_size=5, stride=1, padding=0),                    \n                    nn.ReLU(),                \n                   ]\n        self.cnn = nn.Sequential(*cnn_list)\n    \n        fc_list = [nn.Linear(120, 84),\n                   nn.ReLU(),\n                   nn.BatchNorm1d(84),\n                   nn.Dropout(p=0.2),\n                   nn.Linear(84,10),\n                  ]\n        self.fc = nn.Sequential(*fc_list)\n        \n    def forward(self, x):\n        x = self.cnn(x)\n        x = x.view(x.size(0),-1)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Accuracy function**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy_cnt_(y_pred, y_tar, total_cnt=None, match_cnt=None):\n    y_idx = torch.argmax(y_pred, dim=1)\n    ckt = y_idx - y_tar\n    cnt = 0\n    total = 0\n    for i in ckt:\n        total +=1\n        if i == 0:\n            cnt +=1 \n    \n    if total_cnt is not None:\n        total_cnt.append(total)\n        match_cnt.append(cnt)\n        return total_cnt, match_cnt\n    else:\n        return total, cnt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**train/valid function**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(dataloader, model, criterion, optimizer):\n        \n    model.train()\n    total, match, loss_list = [], [], []\n    for x, y_tar in dataloader:\n        \n        if torch.cuda.is_available():\n            x = x.cuda()\n            y_tar = y_tar.cuda()\n        \n        y_pred = model(x)\n        loss = criterion(y_pred, y_tar)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        loss_list.append(loss.item())\n        total, match = accuracy_cnt_(y_pred, y_tar, total, match)\n\n    acc = np.sum(match)/np.sum(total)\n    loss_avg = np.mean(loss_list)\n    \n    return model, acc, loss_avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def valid(dataloader, model, criterion):\n    model.eval()\n    loss_list = []\n    for x, y_tar in dataloader:\n        \n        if torch.cuda.is_available():\n            x = x.cuda()\n            y_tar = y_tar.cuda()\n        \n        y_pred = model(x)\n        loss = criterion(y_pred, y_tar)\n        \n        total, match = accuracy_cnt_(y_pred, y_tar)\n        \n    acc = np.sum(match)/np.sum(total)\n    \n    return model, acc, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(n_epoch, trainloader, model, criterion, optimizer,validloader=None):\n    \n    # row(0): epoch no., row(1): acc, row(2): loss\n    train_ls = np.zeros((3, n_epoch))\n    valid_ls = np.zeros((3, n_epoch))\n    valid_acc = 0\n    for epoch in range(n_epoch):\n        model, acc, loss = train(trainloader, model, criterion, optimizer)\n        print('Epoch[{:2d}/{}]: train accuracy={:.2f}% || train loss={:.3f} ||'.format(epoch+1, n_epoch, acc*100, loss), end=' ')\n        \n        train_ls[0][epoch] = epoch+1\n        train_ls[1][epoch] = acc\n        train_ls[2][epoch] = loss\n        \n        \n\n        with torch.no_grad():\n            model, acc, loss = valid(validloader, model, criterion)  \n            print('valid accuracy={:.2f}% || valid loss={:.3f}'.format(acc*100, loss))\n    \n            valid_ls[0][epoch] = epoch+1\n            valid_ls[1][epoch] = acc\n            valid_ls[2][epoch] = loss    \n        \n            if acc > valid_acc:\n                valid_acc = acc\n                best_model = model\n        \n    return best_model, valid_acc, train_ls, valid_ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**parameter, dataloader, and train loop**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data set\ntrainset = mnist_dataset(train_x, train_y, transform = False)\nvalidset = mnist_dataset(valid_x, valid_y, transform = False)\n\n# parameters\nbs = 32 # batch size\nn_epoch = 20\nlr = 0.001 # learning\n\n# data loader\ntrainloader = DataLoader(trainset, bs, shuffle=True)\nvalidloader = DataLoader(validset, len(validset), shuffle=False)\n\n#model = LeNet_()\nmodel = LeNet_mdf() # CNN more feature + ReLU and FC + batch normalization + ReLU\n#model = LeNet_mdf2() # CNN more feature + ReLU and FC ReLU + IC layer(batch normalization + dropout)\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\n#L2_lambda = 0.001\n\n# check if GPU available\nif torch.cuda.is_available():\n    model = model.cuda()\n    criterion = criterion.cuda()\n\n_, _, train_ls, valid_ls = train_loop(n_epoch, trainloader, model, criterion, optimizer, validloader)\n\ntrainset = mnist_dataset(train_x, train_y, transform = True)\nbest_model2, valid_acc2, train_ls2, valid_ls2 = train_loop(n_epoch, trainloader, model, criterion, optimizer, validloader)\n\nmodel = best_model2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot acc and loss\ndef plot_acc_loss_(train_ls, valid_ls):\n    fig, (ax1, ax2) = plt.subplots(1, 2)\n    fig.set_figheight(3)\n    fig.set_figwidth(12)\n    ax1.plot(train_ls[0], train_ls[1], label='train')\n    ax1.plot(valid_ls[0], valid_ls[1], label='valid')\n    ax1.set_title('accuracy - epoch')\n    ax1.set_ylabel('accuracy')\n    ax1.set_xlabel('epoch')\n    ax1.legend()\n    ax2.plot(train_ls[0], train_ls[2], label='train')\n    ax2.plot(valid_ls[0], valid_ls[2], label='valid')\n    ax2.set_title('loss - epoch')\n    ax2.set_ylabel('loss')\n    ax2.set_xlabel('epoch')\n    ax2.legend()\n    plt.show()\n    plt.close()\n\nplot_acc_loss_(train_ls, valid_ls)\nplot_acc_loss_(train_ls2, valid_ls2)\n\ndel train_ls, valid_ls, train_ls2, valid_ls2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**predict test data for semi-supervised learning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"testset  = mnist_dataset(test_X, label = None, transform = False)\ntestloader = DataLoader(testset, bs, shuffle=False)\n\ntest_pred = torch.LongTensor()\n\nfor x in testloader:\n    if torch.cuda.is_available():\n        x = x.cuda()\n    \n    y_pred = model(x)\n    pred = torch.argmax(y_pred.cpu(), dim=1)\n    test_pred = torch.cat((test_pred, pred)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x = np.concatenate((train_x, test_X), axis=0)\ntrain_y = np.concatenate((train_y, np.array(test_pred.numpy())), axis=None)\n\ntrainset = mnist_dataset(train_x, train_y, transform = False)\ntrainloader = DataLoader(trainset, bs, shuffle=True)\n\ndel train_x, train_y, trainset, test_pred, y_pred, pred\n\nbest_model3, valid_acc3, train_ls3, valid_ls3 = train_loop(n_epoch, trainloader, model, criterion, optimizer, validloader)\n\nif (valid_acc3 > valid_acc2):\n    model = best_model3\nelse:\n    model = best_model2\n\nplot_acc_loss_(train_ls3, valid_ls3)\n\ndel train_ls3, valid_ls3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prediction**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# test dataloader\n\ntest_pred = torch.LongTensor()\n\nfor x in testloader:\n    if torch.cuda.is_available():\n        x = x.cuda()\n    y_pred = model(x)\n    pred = torch.argmax(y_pred.cpu(), dim=1)\n    test_pred = torch.cat((test_pred, pred))\n\nidx = []\nfor i in range(len(test_pred.tolist())):\n    idx.append(i+1)\n\npred_dict = {\n                \"Imageid\": idx,\n                \"Label\": test_pred.tolist()\n}\n\nout_df = pd.DataFrame(pred_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(out_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}