{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction.\nThe CIFAR-10 dataset contains 60,000 color images of 32 x 32 pixels in 3 channels divided into 10 classes. Each class contains 6,000 images. The training set contains 50,000 images, while the test sets provides 10,000 images. This image taken from the CIFAR repository ( https://www.cs.toronto.edu/~kriz/cifar.html ).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Import And Preprocess Data ","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from __future__ import print_function\nimport keras\nfrom keras.datasets import cifar10\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nimport os\n\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport itertools\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Downloading DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# The data, split between train and test sets:\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\nprint('x_train shape:', x_train.shape)\nprint('y_train shape:', y_train.shape)\nprint(x_train.shape[0], 'train samples')\nprint(x_test.shape[0], 'test samples')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Count plot for training set\nsns.countplot(y_train.ravel())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Count plot for test set\nsns.countplot(y_test.ravel())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# it is a numpy array\nx_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using show to convert numpy array to image\nplt.imshow(x_train[5])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize the data. Before we need to connvert data type to float for computation.\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train /= 255\nx_test /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert class vectors to binary class matrices. This is called one hot encoding.\nnum_classes = 10  \ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.to_categorical(y_test, num_classes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trying 1st Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D,MaxPooling2D,\\\n     Dropout,Flatten,Dense,Activation,\\\n     BatchNormalization\n\n# Keras defines a sequential model as a sequential stack of linear layers.\nmodel=Sequential()\n\n# here we using 32 filter layers and kernel size of (3,3), with relu activation function (max(0,x))\n# Batch normalization() is a technique for improving the speed, performance, and stability of cnn.\n# max pooling reduces the dimensionality of images by reducing the number of pixels in the output from previous cnn\n# pool size define size of filter\n# Droupout drops out the nodes from the layer and reduce chances of overfitting\n\n#1st layer is input layer\nmodel.add(Conv2D(32,(3,3),activation='relu',input_shape=(32, 32, 3)))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n# 2nd layer\nmodel.add(Conv2D(64,(3,3),activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n# 3rd layer\nmodel.add(Conv2D(128,(3,3),activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n# Flatten convert matrix into single array\nmodel.add(Flatten())\nmodel.add(Dense(512,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\n\n# Finally we compile our model with rsmprop optimizer\nmodel.add(Dense(10,activation='softmax'))\nmodel.compile(loss='categorical_crossentropy',\n  optimizer='rmsprop',metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\n#One way to avoid overfitting is to terminate the process early using earlystop\nearlystop = EarlyStopping(patience = 10)\n\n# Following line will reduce learning rate when a metric has stopped improving. \nlearning_rate_reduction = ReduceLROnPlateau(monitor = 'val_acc',patience = 2,verbose = 1,factor = 0.5,min_lr = 0.00001)\n\ncallbacks = [earlystop,learning_rate_reduction]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nhistory = model.fit(x_train, y_train,\n              batch_size=32,\n              epochs=20,\n              validation_data=(x_test, y_test),\n              callbacks=callbacks,\n                    shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\n\nplotmodelhistory(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trying 2nd Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#define the convnet\nmodel1 = Sequential()\n# CONV => RELU => CONV => RELU => POOL => DROPOUT\nmodel1.add(Conv2D(32, (3, 3), padding='same',input_shape=x_train.shape[1:]))\nmodel1.add(Activation('relu'))\nmodel1.add(Conv2D(32, (3, 3)))\nmodel1.add(Activation('relu'))\nmodel1.add(MaxPooling2D(pool_size=(2, 2)))\nmodel1.add(Dropout(0.25))\n\n# CONV => RELU => CONV => RELU => POOL => DROPOUT\nmodel1.add(Conv2D(64, (3, 3), padding='same'))\nmodel1.add(Activation('relu'))\nmodel1.add(Conv2D(64, (3, 3)))\nmodel1.add(Activation('relu'))\nmodel1.add(MaxPooling2D(pool_size=(2, 2)))\nmodel1.add(Dropout(0.25))\n\n# FLATTERN => DENSE => RELU => DROPOUT\nmodel1.add(Flatten())\nmodel1.add(Dense(512))\nmodel1.add(Activation('relu'))\nmodel1.add(Dropout(0.5))\n# a softmax classifier\nmodel1.add(Dense(num_classes))\nmodel1.add(Activation('softmax'))\n\nopt = keras.optimizers.RMSprop(learning_rate=0.0001, decay=1e-6)\n# Let's train the model using RMSprop\nmodel1.compile(loss='categorical_crossentropy',\n              optimizer=opt,\n              metrics=['accuracy'])\n\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = model1.fit(x_train, y_train,\n              batch_size=32,\n              epochs=20,\n              validation_data=(x_test, y_test),\n                      callbacks=callbacks,\n                    shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotmodelhistory(history1): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history1.history['accuracy']) \n    axs[0].plot(history1.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history1.history['loss']) \n    axs[1].plot(history1.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history1.history.keys())\n\nplotmodelhistory(history1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Score trained model.\nscores = model.evaluate(x_test, y_test, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])\n\n# make prediction.\npred = model.predict(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Score trained model.\nscores = model1.evaluate(x_test, y_test, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])\n\n# make prediction.\npred = model.predict(x_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## We saw that our 2nd model with more parameters performs well on the given dataset. But for more better results use around 40-60 epoch.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Viewing our result via images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(pred, axis=1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_test, axis=1)\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = pred[errors]\nY_true_errors = Y_true[errors]\nX_test_errors = x_test[errors]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(Y_true, Y_pred_classes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# viewing first 25 images with their actual and predicted values\nR = 5    #row\nC = 5    #col\nfig, axes = plt.subplots(R, C, figsize=(12,12))\naxes = axes.ravel()\n\nfor i in np.arange(0, R*C):\n    axes[i].imshow(x_test[i])\n    axes[i].set_title(\"True: %s \\nPredict: %s\" % (labels[Y_true[i]], labels[Y_pred_classes[i]]))\n    axes[i].axis('off')\n    plt.subplots_adjust(wspace=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# viewing wrongly classified images\nR = 3\nC = 5\nfig, axes = plt.subplots(R, C, figsize=(12,8))\naxes = axes.ravel()\n\nmisclassified_idx = np.where(Y_pred_classes != Y_true)[0]\nfor i in np.arange(0, R*C):\n    axes[i].imshow(x_test[misclassified_idx[i]])\n    axes[i].set_title(\"True: %s \\nPredicted: %s\" % (labels[Y_true[misclassified_idx[i]]], \n                                                  labels[Y_pred_classes[misclassified_idx[i]]]))\n    axes[i].axis('off')\n    plt.subplots_adjust(wspace=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}