{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import print_function\nimport keras\nfrom keras.datasets import cifar10\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nimport os\n\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport itertools\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-23T08:16:13.273734Z","iopub.execute_input":"2022-02-23T08:16:13.274259Z","iopub.status.idle":"2022-02-23T08:16:13.285060Z","shell.execute_reply.started":"2022-02-23T08:16:13.274220Z","shell.execute_reply":"2022-02-23T08:16:13.284329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128  # The default batch size of keras.\nnum_classes = 10  # Number of class for the dataset\nepochs = 100\ndata_augmentation = False","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:13.287008Z","iopub.execute_input":"2022-02-23T08:16:13.287275Z","iopub.status.idle":"2022-02-23T08:16:13.297866Z","shell.execute_reply.started":"2022-02-23T08:16:13.287241Z","shell.execute_reply":"2022-02-23T08:16:13.296996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The data, split between train and test sets:\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\nprint('x_train shape:', x_train.shape)\nprint('y_train shape:', y_train.shape)\nprint(x_train.shape[0], 'train samples')\nprint(x_test.shape[0], 'test samples')","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:13.299721Z","iopub.execute_input":"2022-02-23T08:16:13.299948Z","iopub.status.idle":"2022-02-23T08:16:14.041343Z","shell.execute_reply.started":"2022-02-23T08:16:13.299922Z","shell.execute_reply":"2022-02-23T08:16:14.038418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1,2,figsize=(15,5)) \n# Count plot for training set\nsns.countplot(y_train.ravel(), ax=axs[0])\naxs[0].set_title('Distribution of training data')\naxs[0].set_xlabel('Classes')\n# Count plot for testing set\nsns.countplot(y_test.ravel(), ax=axs[1])\naxs[1].set_title('Distribution of Testing data')\naxs[1].set_xlabel('Classes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:14.044518Z","iopub.execute_input":"2022-02-23T08:16:14.044981Z","iopub.status.idle":"2022-02-23T08:16:14.371141Z","shell.execute_reply.started":"2022-02-23T08:16:14.044946Z","shell.execute_reply":"2022-02-23T08:16:14.370456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import np_utils\n\n# Normalize the data. Before we need to connvert data type to float for computation.\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train /= 255\nx_test /= 255\n\n# Convert class vectors to binary class matrices. This is called one hot encoding.\ny_train = keras.utils.np_utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.np_utils.to_categorical(y_test, num_classes)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:14.372975Z","iopub.execute_input":"2022-02-23T08:16:14.373727Z","iopub.status.idle":"2022-02-23T08:16:14.624981Z","shell.execute_reply.started":"2022-02-23T08:16:14.373687Z","shell.execute_reply":"2022-02-23T08:16:14.624239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), padding = 'SAME', input_shape = x_train.shape[1:]))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size = (2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, (3, 3), padding = 'SAME'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size = (2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(512))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes))\nmodel.add(Activation('softmax'))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:14.626250Z","iopub.execute_input":"2022-02-23T08:16:14.626546Z","iopub.status.idle":"2022-02-23T08:16:14.718609Z","shell.execute_reply.started":"2022-02-23T08:16:14.626498Z","shell.execute_reply":"2022-02-23T08:16:14.717953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import keras\nfrom keras import optimizers\n\nopt = keras.optimizers.Adam(learning_rate = 0.001)\n\nmodel.compile(loss = 'categorical_crossentropy',\n             optimizer = opt,\n             metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:14.721185Z","iopub.execute_input":"2022-02-23T08:16:14.721373Z","iopub.status.idle":"2022-02-23T08:16:14.730109Z","shell.execute_reply.started":"2022-02-23T08:16:14.721350Z","shell.execute_reply":"2022-02-23T08:16:14.729396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = None  # For recording the history of trainning process.\nif not data_augmentation:\n    print('Not using data augmentation.')\n    history = model.fit(x_train, y_train,\n              batch_size=batch_size,\n              epochs=epochs,\n              validation_data=(x_test, y_test),\n              shuffle=True)\nelse:\n    print('Using real-time data augmentation.')\n    # This will do preprocessing and realtime data augmentation:\n    datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n        # randomly shift images horizontally (fraction of total width)\n        width_shift_range=0.1,\n        # randomly shift images vertically (fraction of total height)\n        height_shift_range=0.1,\n        shear_range=0.,  # set range for random shear\n        zoom_range=0.,  # set range for random zoom\n        channel_shift_range=0.,  # set range for random channel shifts\n        # set mode for filling points outside the input boundaries\n        fill_mode='nearest',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=False,  # randomly flip images\n        # set rescaling factor (applied before any other transformation)\n        rescale=None,\n        # set function that will be applied on each input\n        preprocessing_function=None,\n        # image data format, either \"channels_first\" or \"channels_last\"\n        data_format=None,\n        # fraction of images reserved for validation (strictly between 0 and 1)\n        validation_split=0.0)\n\n    # Compute quantities required for feature-wise normalization\n    # (std, mean, and principal components if ZCA whitening is applied).\n    datagen.fit(x_train)\n\n    # Fit the model on the batches generated by datagen.flow().\n    history = model.fit_generator(datagen.flow(x_train, y_train,\n                                    batch_size=batch_size),\n                                    epochs=epochs,\n                                    validation_data=(x_test, y_test),\n                                    workers=4)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:16:14.731468Z","iopub.execute_input":"2022-02-23T08:16:14.731909Z","iopub.status.idle":"2022-02-23T08:21:10.837562Z","shell.execute_reply.started":"2022-02-23T08:16:14.731870Z","shell.execute_reply":"2022-02-23T08:21:10.836873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plotmodelhistory(history):\n    fig, ax = plt.subplots(1, 2, figsize = (15, 5))\n    ax[0].plot(history.history['accuracy'])\n    ax[0].plot(history.history['val_accuracy'])\n    ax[0].set_title('Model Accuracy')\n    ax[0].set_ylabel('Accuracy')\n    ax[0].set_xlabel('Epoch')\n    ax[0].legend(['train', 'validate'], loc = 'upper left')\n    \n    ax[1].plot(history.history['loss'])\n    ax[1].plot(history.history['val_loss'])\n    ax[1].set_title('Model Loss')\n    ax[1].set_ylabel('Loss')\n    ax[1].set_xlabel('Epoch')\n    ax[1].legend(['train', 'validate'], loc = 'upper left')\n    plt.show()\n    \nprint(history.history.keys())\n\nplotmodelhistory(history)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:24:40.469389Z","iopub.execute_input":"2022-02-23T08:24:40.469654Z","iopub.status.idle":"2022-02-23T08:24:40.828727Z","shell.execute_reply.started":"2022-02-23T08:24:40.469625Z","shell.execute_reply":"2022-02-23T08:24:40.827922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.evaluate(x_test, y_test, verbose = 1)\nprint('Test loss : ', scores[0])\nprint('Test accuracy : ', scores[1])\n\npred = model.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:25:37.878793Z","iopub.execute_input":"2022-02-23T08:25:37.879345Z","iopub.status.idle":"2022-02-23T08:25:40.491113Z","shell.execute_reply.started":"2022-02-23T08:25:37.879309Z","shell.execute_reply":"2022-02-23T08:25:40.490356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def heatmap(data, row_labels, col_labels, ax = None, cbar_kw = {}, cbarlabel = '', **kwargs):\n    \n    if not ax:\n        ax = plt.gca()\n    im = ax.imshow(data, **kwargs)\n    \n    cbar = ax.figure.colorbar(im, ax= ax, **cbar_kw)\n    cbar.ax.set_ylabel(cbarlabel, rotation = -90, va = 'bottom')\n    \n    ax.tick_params(top = True, bottom = False, labeltop = True, labelbottom = False)\n    \n\n    ax.set_xticks(np.arange(data.shape[1]))\n    ax.set_yticks(np.arange(data.shape[0]))\n    \n    ax.set_xticklabels(col_labels)\n    ax.set_yticklabels(row_labels)\n    \n    ax.set_xlabel('Predicted Label')\n    ax.set_ylabel('True Label')\n    \n    return im, cbar\n\ndef annotate_heatmap(im, data = None, fmt = 'd', threshold = None):\n    \n    texts = []\n    for i in range(data.shape[0]):\n        for j in range(data.shape[1]):\n            text = im.axes.text(j, i, format(data[i, j], fmt), horizontalalignment = 'center',\n                               color = 'white' if data[i, j] > thresh else 'black')\n            \n            texts.append(text)\n            \n    return texts","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:33:29.621488Z","iopub.execute_input":"2022-02-23T08:33:29.621767Z","iopub.status.idle":"2022-02-23T08:33:29.647834Z","shell.execute_reply.started":"2022-02-23T08:33:29.621737Z","shell.execute_reply":"2022-02-23T08:33:29.646890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n\ny_pred_classes = np.argmax(pred, axis = 1)\ny_true = np.argmax(y_test, axis = 1)\nerrors = (y_pred_classes - y_true != 0)\n\ny_pred_classes_errors = y_pred_classes[errors]\ny_pred_errors = pred[errors]\ny_true_errors = y_true[errors]\nx_test_errors = x_test[errors]\n\ncm = confusion_matrix(y_true, y_pred_classes)\nthresh = cm.max() / 2,\n\nfig, ax = plt.subplots(figsize = (12, 12))\nim, cbar = heatmap(cm, labels, labels, ax = ax,\n                  cmap = plt.cm.Blues, cbarlabel = 'count of predictions')\n\ntexts = annotate_heatmap(im, data = cm, threshold = thresh)\n\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:36:34.753041Z","iopub.execute_input":"2022-02-23T08:36:34.753575Z","iopub.status.idle":"2022-02-23T08:36:35.635439Z","shell.execute_reply.started":"2022-02-23T08:36:34.753537Z","shell.execute_reply":"2022-02-23T08:36:35.634660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_true, y_pred_classes))","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:36:55.358126Z","iopub.execute_input":"2022-02-23T08:36:55.358655Z","iopub.status.idle":"2022-02-23T08:36:55.386414Z","shell.execute_reply.started":"2022-02-23T08:36:55.358617Z","shell.execute_reply":"2022-02-23T08:36:55.385536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"R = 5\nC = 5\nfig, axes = plt.subplots(R, C, figsize = (12, 12))\naxes = axes.ravel()\n\nfor i in np.arange(0, R*C):\n    axes[i].imshow(x_test[i])\n    axes[i].set_title('True: %s \\nPredict: %s' % (labels[y_true[i]], labels[y_pred_classes[i]]))\n    axes[i].axis('off')\n    plt.subplots_adjust(wspace = 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:38:10.028535Z","iopub.execute_input":"2022-02-23T08:38:10.029007Z","iopub.status.idle":"2022-02-23T08:38:11.604414Z","shell.execute_reply.started":"2022-02-23T08:38:10.028963Z","shell.execute_reply":"2022-02-23T08:38:11.601144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"R = 3\nC = 5\nfig, axes = plt.subplots(R, C, figsize = (12, 8))\naxes = axes.ravel()\n\nmisclassified_idx = np.where(y_pred_classes != y_true)[0]\nfor i in np.arange(0, R*C):\n    axes[i].imshow(x_test[misclassified_idx[i]])\n    axes[i].set_title('True: %s \\nPredict: %s' % (labels[y_true[misclassified_idx[i]]], \n                                                  labels[y_pred_classes[misclassified_idx[i]]]))\n    axes[i].axis('off')\n    plt.subplots_adjust(wspace = 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:39:32.621456Z","iopub.execute_input":"2022-02-23T08:39:32.622185Z","iopub.status.idle":"2022-02-23T08:39:44.101430Z","shell.execute_reply.started":"2022-02-23T08:39:32.622142Z","shell.execute_reply":"2022-02-23T08:39:44.100710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_errors(errors_index, img_errors, pred_errors, obs_errors):\n    n = 0\n    nrows = 2\n    ncols = 5\n    fig, ax = plt.subplots(nrows, ncols, sharex = True, sharey = True, figsize = (12, 6))\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row, col].imshow((img_errors[error]).reshape((32, 32, 3)))\n            ax[row, col].set_title('Predicted:{}\\nTrue:{}'.\n                                  format(labels[pred_errors[error]], labels[obs_errors[error]]))\n            n += 1\n            ax[row, col].axis('off')\n            plt.subplots_adjust(wspace = 1)\n        \ny_pred_errors_prob = np.max(y_pred_errors, axis = 1)\ntrue_prob_errors = np.diagonal(np.take(y_pred_errors, y_true_errors, axis = 1))\ndelta_pred_true_errors = y_pred_errors_prob - true_prob_errors\n\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\nmost_important_errors = sorted_dela_errors[-10:]\n\ndisplay_errors(most_important_errors, x_test_errors, y_pred_classes_errors, y_true_errors)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:44:22.506565Z","iopub.execute_input":"2022-02-23T08:44:22.506840Z","iopub.status.idle":"2022-02-23T08:44:23.037315Z","shell.execute_reply.started":"2022-02-23T08:44:22.506809Z","shell.execute_reply":"2022-02-23T08:44:23.036587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_test(number):\n    fig = plt.figure(figsize = (3, 3))\n    test_image = np.expand_dims(x_test[number], axis = 0)\n    test_prob = model.predict(test_image, verbose = 0)\n    test_result = test_prob.argmax(axis = -1)\n    plt.imshow(x_test[number])\n    dict_key = test_result[0]\n    plt.title('Predicted : {} \\nTrue Label : {}'.format(labels[dict_key],\n                                                       labels[y_true[number]]))","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:50:05.952091Z","iopub.execute_input":"2022-02-23T08:50:05.952703Z","iopub.status.idle":"2022-02-23T08:50:05.965426Z","shell.execute_reply.started":"2022-02-23T08:50:05.952653Z","shell.execute_reply":"2022-02-23T08:50:05.964481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_test(20)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T08:50:10.650117Z","iopub.execute_input":"2022-02-23T08:50:10.650909Z","iopub.status.idle":"2022-02-23T08:50:10.930797Z","shell.execute_reply.started":"2022-02-23T08:50:10.650859Z","shell.execute_reply":"2022-02-23T08:50:10.930036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}