{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport scipy\nfrom scipy.io import arff\nimport pandas as pd\nimport numpy as np\nimport pickle\nimport matplotlib.pyplot as plt\nimport IPython\nimport time\nimport sys\n\nimport sklearn\nfrom sklearn import preprocessing\nfrom sklearn import datasets \nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nfrom sklearn.decomposition import PCA\nfrom sklearn import tree\nfrom sklearn.tree import export_graphviz\nfrom sklearn.manifold import TSNE\n\nimport torch\nfrom torch.nn import Linear, ReLU, CrossEntropyLoss, Sequential, Conv2d, MaxPool2d, Module, Softmax, Dropout2d\nimport torch.optim as optim\nimport torchvision\nfrom torchvision.transforms import *\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data sets path\n# DATA_PATH = '/kaggle/input/cifar10/'\nDATA_PATH = 'dataset/'\nIMAGE_PATH = 'image/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print progress bar\ndef progress(count, total, suffix=''):\n    bar_len = 30\n    filled_len = int(round(bar_len * count / float(total)))\n    \n    percents = 100 * count // total\n    bar = '=' * filled_len +'>'+ '.' * (bar_len - filled_len)\n    if percents == 100:\n        bar = '=' * filled_len +'='+ '.' * (bar_len - filled_len)\n    length = len(str(total))\n    sys.stdout.write('%s/%s [%s]%s%% %s\\r' % (('{:>%s}'%length).format(count), total, bar, '{:>3}'.format(percents), suffix))\n    sys.stdout.flush()\n\n#plot images    \ndef plot_image(X):\n    plt.figure(figsize=(50,5))\n    for i,x in enumerate(X[:10]):\n        plt.subplot(1,10,i+1)\n        try:\n            plt.imshow(np.array(x))\n        except:\n            plt.imshow(np.array(x).transpose(1,2,0))\n    \ndef dimension_reduction(x_train, x_test, n_components):\n    print(\"Reducting dimensions...\")\n    pca = PCA(n_components=n_components, random_state=33)\n    pca.fit(x_train)\n    x_train= pca.transform(x_train)\n    x_test = pca.transform(x_test)\n    return x_train, x_test, pca\n\n#load data from local disk\ndef load_CIFAR_10(training_batches=5, test_batches=1):\n    X_train=y_train=X_test=y_test = None\n    for b in range(1,training_batches+1):\n        with open(DATA_PATH+'data_batch_%s'%b,'rb') as file:\n            data = pickle.load(file,encoding='bytes')\n            X = np.array(data[b'data'])\n            y = np.array(data[b'labels'])\n            try:\n                X_train = np.concatenate((X_train,X))\n                y_train = np.concatenate((y_train,y))\n                print('train set batch: %d'%b)\n            except:\n                print('train set batch: %d'%b)\n                X_train = X\n                y_train = y\n                \n    for b in range(1,test_batches+1):\n        with open(DATA_PATH+'test_batch','rb') as file:\n            data = pickle.load(file,encoding='bytes')\n            X = np.array(data[b'data'])\n            y = np.array(data[b'labels'])\n            try:\n                X_test = np.concatenate((X_test,X))\n                y_test = np.concatenate((y_test,y))\n            except:\n                print('test set batch: %d'%b)\n                X_test = X\n                y_test = y\n    return X_train,y_train,X_test,y_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PCA Projection\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def scatter_two_component(comp1,comp2,y_train,classes,save_as):\n    plt.figure(figsize=(20,15))\n    color_map = plt.cm.get_cmap('Accent')\n    \n    #plot without labels (faster)\n    plt.scatter(comp1,comp2,c=y_train,cmap=color_map)\n\n    #plot labels\n    labels = np.array(classes)[y_train]\n    class_num = set()\n    for x1,x2,c,l in zip(comp1,comp2,color_map(y_train),labels):\n        if len(class_num)==10:\n            break\n        plt.scatter(x1,x2,c=[c],label=l)\n        class_num.add(l)\n        \n    #remvoe duplicate labels    \n    hand, labl = plt.gca().get_legend_handles_labels()\n    handout=[]\n    lablout=[]\n    for h,l in zip(hand,labl):\n        if l not in lablout:\n            lablout.append(l)\n            handout.append(h)\n    plt.title(save_as)\n    plt.xlabel('Component One')\n    plt.ylabel('Component Two')\n    plt.legend(handout, lablout,fontsize=20)\n    plt.savefig(IMAGE_PATH+save_as)\n    plt.show()\n\n# X_train,y_train,X_test,y_test = load_CIFAR_10()\n# X_train,X_test,pca = dimension_reduction(X_train,X_test,n_components=50)\n\n# pca.explained_variance_ratio_\n\n# scatter_two_component(X_train[:,0],X_train[:,1],y_train,classes,'PCA')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# t-SNE Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# try:\n#     with open('X_embedded','rb') as file:\n#         X_embedded = pickle.load(file)\n# except:\n#     X_embedded = TSNE(n_components=2).fit_transform(X_train)\n#     with open('X_embedded','wb') as file:\n#         pickle.dump(X_embedded,file)\n\n# scatter_two_component(X_embedded[:,0],X_embedded[:,1],y_train,classes,'t-SNE')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train,y_train,X_test,y_test = load_CIFAR_10()\n\nX_train = torch.tensor(X_train.reshape(-1,3,32,32))\nX_test = torch.tensor(X_test.reshape(-1,3,32,32))\ny_train = torch.tensor(y_train)\ny_test = torch.tensor(y_test)\nassert type(X_train)==torch.Tensor,\"Type(X_train) == Torch.tensor is required, got %s\"%type(X_train)\nC = X_train.shape[1]\nH = X_train.shape[2]\nW = X_train.shape[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tranform functions\nT = Compose([\n    ToPILImage(),\n#     CenterCrop(32),\n#     ColorJitter(brightness=1, contrast=0, saturation=0, hue=0),\n#     Pad(2, fill=(100,100,100), padding_mode='constant'),\n#     Grayscale(3),    \n    RandomHorizontalFlip(p=0.5),\n#     RandomAffine(30, translate=None, scale=None, shear=None, resample=False, fillcolor=0),\n#     RandomCrop(32, padding=None, pad_if_needed=False, fill=0, padding_mode='constant'),\n#     RandomPerspective(distortion_scale=0.3, p=0.5, interpolation=3),\n#     RandomRotation(10, resample=False, expand=False, center=None),\n    RandomResizedCrop(32, scale=(0.75, 1), ratio=(0.75, 1), interpolation=2),\n    ToTensor(),\n    Normalize((0.4753,0.4623,0.4149), (0.2414,0.236,0.2419)),\n#     Normalize((0.5,0.5,0.5), (0.5,0.5,0.5)),\n])\n\n# tranform functions\nT2 = Compose([\n    ToPILImage(),\n#     Grayscale(3),\n    ToTensor(),\n    Normalize((0.4753,0.4623,0.4149), (0.2414,0.236,0.2419)),\n#     Normalize((0.5,0.5,0.5), (0.5,0.5,0.5)),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_new = torch.stack([T(x) for x in X_train])\nX_train = torch.stack([T2(x) for x in X_train])\nX_test = torch.stack([T2(x) for x in X_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#augmentation\nX_train = torch.stack([X_new,X_train]).reshape(-1,C,H,W)\ny_train = torch.stack([y_train,y_train]).reshape(-1,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cnn parameters\nimage_channels = X_train.shape[1]\nimage_width = X_train.shape[2]\nnum_filters = 32\nnum_filters2 = 64\nnum_filters3 = 128\nfilter_size = 5\npool_size = 2\n# final_input = (((image_width+1-filter_size)//pool_size+1-filter_size)//pool_size)**2*num_filters2#without padding\n# final_input = (image_width//pool_size//pool_size//pool_size)**2*num_filters3#with padding\n\nclass net(Module):   \n    def __init__(self):\n        super(net, self).__init__()\n        self.final_input = 2048\n        self.cnn_layers = Sequential(\n            Conv2d(image_channels, num_filters, filter_size, padding=filter_size//2),\n            ReLU(),\n            MaxPool2d(pool_size, pool_size),\n\n            Conv2d(num_filters, num_filters2, filter_size, padding=filter_size//2),\n            ReLU(),\n            MaxPool2d(pool_size, pool_size),\n\n            Conv2d(num_filters2, num_filters3, filter_size, padding=filter_size//2),\n            ReLU(),\n            MaxPool2d(pool_size, pool_size),\n        )\n\n        self.linear_layers = Sequential(\n            Linear(self.final_input, self.final_input//2),\n            ReLU(),\n            Dropout2d(p=0.5),\n\n            Linear(self.final_input//2, self.final_input//4),\n            ReLU(),\n            Dropout2d(p=0.5),\n\n            Linear(self.final_input//4, self.final_input//16),\n            ReLU(),\n            Dropout2d(p=0.5),\n\n            Linear(self.final_input//16,10),\n        )\n\n    # Defining the forward pass    \n    def forward(self, x):\n        x = self.cnn_layers(x)\n        x = x.view(-1, self.final_input)\n        x = self.linear_layers(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_model(model,batch_size,test_loader,criterion):\n    correct = 0\n    total = 0\n    running_loss = 0.0\n    with torch.no_grad():\n        for data in test_loader:\n            X, y = data\n            if torch.cuda.is_available():\n                X, y = X.cuda(), y.cuda()\n            y_proba = model(X)\n            loss = criterion(y_proba, y)\n            running_loss += loss.item()\n            _, y_pred = torch.max(y_proba.data, 1)\n            total += y.size(0)\n            correct += (y_pred == y).sum().item()\n    loss = running_loss/len(test_loader)\n    acc = correct/total\n    return loss, acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_class_acc(model,test_loader,classes):\n    class_correct = list(0. for i in range(len(classes)))\n    class_total = list(0. for i in range(len(classes)))\n    with torch.no_grad():\n        for data in test_loader:\n            X, y = data\n            if torch.cuda.is_available():\n                X, y = X.cuda(), y.cuda()\n            y_proba = model(X)\n            _, y_pred = torch.max(y_proba, 1)\n            c = (y_pred == y).squeeze()\n            for i in range(4):\n                label = y[i]\n                class_correct[label] += c[i].item()\n                class_total[label] += 1\n\n    for i in range(len(classes)):\n        print('Accuracy of %5s : %2d %%' % (\n            classes[i], 100 * class_correct[i] / class_total[i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_prediction(test_loader):\n    with open('results.csv','bw') as file:\n        file.write(\"ImageId,Label\\n\".encode())\n        with torch.no_grad():\n            for i, data in enumerate(test_loader, 1):\n                X = data[0]\n                if torch.cuda.is_available():\n                    X = X.cuda()\n                y_proba = model(X)\n                _, y_pred = torch.max(y_proba.data, 1)\n                pair = '%s,%s\\n'%(i,y_pred.item())\n                file.write(pair.encode())\n    print(\"Prediction saved as results.csv\")","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"def train_model(model,batch_size,num_epoch,criterion,optimizer,train_loader,test_loader):\n    N = N_test = 0\n    test_label = False\n    for x in train_loader:\n        N += x[0].size(0)\n    for x in test_loader:\n        N_test += x[0].size(0)\n    for x in test_loader:\n        if len(x) == 2:\n            test_label = True\n            break\n    print('Train on %s samples, Test on %s samples'%(N,N_test))\n    \n    for epoch in range(num_epoch):\n        print('Epoch: %s/%s'%(epoch+1,num_epoch))\n        total = 0\n        correct = 0\n        running_loss = 0.0\n        start_time = time.time()\n        for i, data in enumerate(train_loader, 1):\n            X, y = data\n            if torch.cuda.is_available():\n                X, y = X.cuda(), y.cuda()\n            optimizer.zero_grad()\n\n            # forward + backward + optimize\n            y_proba = model(X)\n            \n            #avg batch acc\n            _, y_pred = torch.max(y_proba.data, 1)\n            total += y.size(0)\n            correct += (y_pred == y).sum().item()\n            \n            loss = criterion(y_proba, y)\n            loss.backward()\n            optimizer.step()\n\n            #avg batch loss\n            running_loss += loss.item()\n            if i == len(train_loader) and test_label:\n                t_loss,t_acc = test_model(model,batch_size,test_loader,criterion)\n                progress(total,N,' time:%ss, loss: %.4f, acc: %.4f, test_loss: %.4f, test_acc: %.4f'\n                         %('{:>3}'.format('%.f'%float(time.time()-start_time)),running_loss/i,correct/total,t_loss,t_acc))\n            else:\n                progress(total,N,' time:%ss, loss: %.4f, acc: %.4f'%\n                         ('{:>3}'.format('%.f'%float(time.time()-start_time)),running_loss/i,correct/total))\n                \n#         if (epoch+1) % 10 ==0:\n#             torch.save(model.state_dict(), 'model_%s.pt'%(epoch+1))\n        sys.stdout.write('\\n')\n    torch.save(model.state_dict(), 'model.pt')\n    print(model)\n    print('Training Finished')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_fit(model,batch_size,num_epoch,criterion,optimizer,X_train,y_train,X_test,y_test=None,pre_trained_model=None):\n    # create torch Dataset class from tensors\n    train_set = torch.utils.data.TensorDataset(X_train, y_train)\n    train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True)\n    if type(y_test) == type(None):#no y_test available, save predictions to disk\n        test_set = torch.utils.data.TensorDataset(X_test)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False)\n    else:\n        test_set = torch.utils.data.TensorDataset(X_test, y_test)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False)   \n        \n    if pre_trained_model == None:\n        print('Start training new model...')\n        model = train_model(model,batch_size,num_epoch,criterion,optimizer,train_loader,test_loader)\n    #try to load pre-trained model \n    else:\n        try:\n            model.load_state_dict(torch.load(pre_trained_model))\n            print(\"Pre-trained model loaded\")\n        except:\n            print('Model not found, start training new model...')\n            model = train_model(model,batch_size,num_epoch,criterion,optimizer,train_loader,test_loader)\n            \n    if type(y_test) == type(None):#no test labels\n        print(\"Test labels not available, saving predictions to disk...\")\n        save_prediction(test_loader)\n    else:\n        loss,acc = test_model(model,batch_size,test_loader,criterion)\n        print('Accuracy of the network on the test set: %.2f%%, loss: %.4f' % (acc*100, loss))\n        test_class_acc(model,test_loader,classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create new model\nmodel = net()\nif torch.cuda.is_available():\n    print('Load model into GPU')\n    moedl = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n'''\n    Rerun this cell to train the same model multiple epoches.\n'''\nbatch_size=128\nnum_epoch=10\ncriterion = CrossEntropyLoss()\n# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-3)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\nclasses = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\nmodel_fit(model,batch_size,num_epoch,criterion,optimizer,X_train,y_train,X_test,y_test,pre_trained_model=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Activation maximization"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"def plot_activation_image(target):\n    zeros = np.zeros((1,3,32,32)).astype(np.float32)\n    gray = zeros+0.5\n    X = torch.tensor(gray, requires_grad=True)\n    target = torch.empty(1, dtype=torch.long).random_(target,target+1)\n    # plot_image(gray)\n    criterion = CrossEntropyLoss()\n\n    for i in range(100):\n        # y probabilities [p0,p1,...,p9]\n        y_proba = model(X)\n\n        #backward\n        loss = criterion(y_proba, target)\n        loss.backward()\n\n        #minimize loss\n        X.data -= 0.2*X.grad.data    \n\n        X.grad.data.zero_()\n        \n#         if i%10 ==0:\n#             IPython.display.display(plt.gcf())       # Tell Jupyter to show Matplotlib's current figure\n#             IPython.display.clear_output(wait=True)  # Tell Jupyter to clear whatever it's currently showing\n#             time.sleep(0.02)\n    plt.title(\"Target class: %s\"%classes[target])\n    plt.imshow(X[0].detach().numpy().transpose(1,2,0))\n#     plt.savefig(IMAGE_PATH+classes[target]+'_activation')\n    print('finished')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model.cpu()\nplt.figure(figsize=(25,10))\nfor i in range(10):\n    plt.subplot(2,5,i+1)\n    plot_activation_image(i)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.5"}},"nbformat":4,"nbformat_minor":1}