{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from __future__ import print_function\nimport keras\nfrom keras.datasets import cifar10\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization\nfrom keras.layers import Conv2D, MaxPooling2D\nimport os\n\nimport numpy as np\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32  # The default batch size of keras.\nnum_classes = 10  # Number of class for the dataset\nepochs = 55","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(x_train, y_train), (x_test, y_test) = cifar10.load_data()\nprint('x_train shape:', x_train.shape)\nprint(x_train.shape[0], 'train samples')\nprint(x_test.shape[0], 'test samples')\n\n# Normalize the data. Before we need to connvert data type to float for computation.\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train /= 255\nx_test /= 255\n\n# Convert class vectors to binary class matrices. This is called one hot encoding.\ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.to_categorical(y_test, num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define the convnet\nmodel = Sequential()\n# CONV => RELU => CONV => RELU => POOL => DROPOUT\nmodel.add(Conv2D(32, (3, 3), padding='same',input_shape=x_train.shape[1:]))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n# CONV => RELU => CONV => RELU => POOL => DROPOUT\nmodel.add(Conv2D(64, (3, 3), padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n# FLATTERN => DENSE => RELU => DROPOUT\nmodel.add(Flatten())\nmodel.add(Dense(512))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\n# a softmax classifier\nmodel.add(Dense(num_classes))\nmodel.add(Activation('softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# initiate RMSprop optimizer\nopt = keras.optimizers.RMSprop(learning_rate=0.0001, decay=1e-6)\n\n# Let's train the model using RMSprop\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=opt,\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndata_augmentation = False # not data augmentation\nhistory = None\nif not data_augmentation:\n    print('Not using data augmentation.')\n    history = model.fit(x_train, y_train,\n              batch_size=batch_size,\n              epochs=epochs,\n              validation_data=(x_test, y_test),\n              shuffle=True)\nelse:\n\n    print('Using real-time data augmentation.')\n    # This will do preprocessing and realtime data augmentation:\n    datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n        # randomly shift images horizontally (fraction of total width)\n        width_shift_range=0.1,\n        # randomly shift images vertically (fraction of total height)\n        height_shift_range=0.1,\n        shear_range=0.,  # set range for random shear\n        zoom_range=0.,  # set range for random zoom\n        channel_shift_range=0.,  # set range for random channel shifts\n        # set mode for filling points outside the input boundaries\n        fill_mode='nearest',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=False,  # randomly flip images\n        # set rescaling factor (applied before any other transformation)\n        rescale=None,\n        # set function that will be applied on each input\n        preprocessing_function=None,\n        # image data format, either \"channels_first\" or \"channels_last\"\n        data_format=None,\n        # fraction of images reserved for validation (strictly between 0 and 1)\n        validation_split=0.0)\n\n    # Compute quantities required for feature-wise normalization\n    # (std, mean, and principal components if ZCA whitening is applied).\n    datagen.fit(x_train)\n\n    # Fit the model on the batches generated by datagen.flow().\n    history = model.fit_generator(datagen.flow(x_train, y_train,\n                                    batch_size=batch_size),\n                                    epochs=epochs,\n                                    validation_data=(x_test, y_test),\n                                    workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\n\nplotmodelhistory(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Score trained model.\nscores = model.evaluate(x_test, y_test, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])\n\n# make prediction.\npred = model.predict(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def heatmap(data, row_labels, col_labels, ax=None, cbar_kw={}, cbarlabel=\"\", **kwargs):\n    \"\"\"\n    Create a heatmap from a numpy array and two lists of labels.\n    \"\"\"\n    if not ax:\n        ax = plt.gca()\n\n    # Plot the heatmap\n    im = ax.imshow(data, **kwargs)\n\n    # Create colorbar\n    cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)\n    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va=\"bottom\")\n\n    # Let the horizontal axes labeling appear on top.\n    ax.tick_params(top=True, bottom=False,\n                   labeltop=True, labelbottom=False)\n    # We want to show all ticks...\n    ax.set_xticks(np.arange(data.shape[1]))\n    ax.set_yticks(np.arange(data.shape[0]))\n    # ... and label them with the respective list entries.\n    ax.set_xticklabels(col_labels)\n    ax.set_yticklabels(row_labels)\n    \n    ax.set_xlabel('Predicted Label') \n    ax.set_ylabel('True Label')\n    \n    return im, cbar\n\ndef annotate_heatmap(im, data=None, fmt=\"d\", threshold=None):\n    \"\"\"\n    A function to annotate a heatmap.\n    \"\"\"\n    # Change the text's color depending on the data.\n    texts = []\n    for i in range(data.shape[0]):\n        for j in range(data.shape[1]):\n            text = im.axes.text(j, i, format(data[i, j], fmt), horizontalalignment=\"center\",\n                                 color=\"white\" if data[i, j] > thresh else \"black\")\n            texts.append(text)\n\n    return texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(pred, axis=1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_test, axis=1)\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = pred[errors]\nY_true_errors = Y_true[errors]\nX_test_errors = x_test[errors]\n\ncm = confusion_matrix(Y_true, Y_pred_classes) \nthresh = cm.max() / 2.\n\nfig, ax = plt.subplots(figsize=(12,12))\nim, cbar = heatmap(cm, labels, labels, ax=ax,\n                   cmap=plt.cm.Blues, cbarlabel=\"count of predictions\")\ntexts = annotate_heatmap(im, data=cm, threshold=thresh)\n\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#NEXT CNN MODEL WITH DATA AUGEMENTATION , EARLY STOPPING , BATCH NORMALIZATION , ADAM OPTIMIZER, learning rate = 0.0005","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define the convnet\nmodel1 = Sequential()\nmodel1.add(Conv2D(32, (3, 3), padding='same',input_shape=x_train.shape[1:]))\nmodel1.add(Activation('relu'))\nmodel1.add(BatchNormalization())\n\nmodel1.add(Conv2D(32, (3, 3)))\nmodel1.add(Activation('relu'))\nmodel1.add(BatchNormalization())\nmodel1.add(MaxPooling2D(pool_size=(2, 2)))\nmodel1.add(Dropout(0.25))\n\nmodel1.add(Conv2D(64, (3, 3), padding='same'))\nmodel1.add(Activation('relu'))\nmodel1.add(BatchNormalization())\n\nmodel1.add(Conv2D(64, (3, 3)))\nmodel1.add(Activation('relu'))\nmodel1.add(BatchNormalization())\nmodel1.add(MaxPooling2D(pool_size=(2, 2)))\nmodel1.add(Dropout(0.25))\n\nmodel1.add(Flatten())\nmodel1.add(Dense(512))\nmodel1.add(Activation('relu'))\nmodel1.add(BatchNormalization())\nmodel1.add(Dropout(0.5))\n\nmodel1.add(Dense(num_classes))\nmodel1.add(Activation('softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = keras.optimizers.Adam(learning_rate=0.0005, decay=1e-6)\n\n# Let's train the model using RMSprop\nmodel1.compile(loss='categorical_crossentropy',\n              optimizer=opt,\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Using real-time data augmentation.')\n    # This will do preprocessing and realtime data augmentation:\ndatagen = ImageDataGenerator(rotation_range=40,width_shift_range=0.2,height_shift_range=0.2,shear_range=0.2,zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')\n\n    # Compute quantities required for feature-wise normalization\n    # (std, mean, and principal components if ZCA whitening is applied).\ndatagen.fit(x_train)\ndatagen.fit(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\nearly = EarlyStopping(patience=12)\nhis = model1.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), validation_data = datagen.flow(x_test, y_test, batch_size=batch_size), epochs=epochs, callbacks = [early],workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotmodelhistory(his): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(his.history['accuracy']) \n    axs[0].plot(his.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(his.history['loss']) \n    axs[1].plot(his.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(his.history.keys())\n\nplotmodelhistory(his)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NEXT CNN MODEL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mode = Sequential()\n\nmode.add(Conv2D(filters = 32,input_shape = (32,32,3),kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(Dropout(0.3))\n\nmode.add(Conv2D(filters = 32,kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(MaxPooling2D(pool_size=(2,2)))\n\nmode.add(Conv2D(filters = 64,kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(Dropout(0.3))\n\nmode.add(Conv2D(filters = 64,kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(MaxPooling2D(pool_size=(2,2)))\n\nmode.add(Conv2D(filters = 128,kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(Dropout(0.3))\n\nmode.add(Conv2D(filters = 128,kernel_size=(3,3),padding = 'same',activation='relu'))\nmode.add(MaxPooling2D(pool_size=(2,2)))\n\nmode.add(Flatten())\nmode.add(Dropout(0.5))\nmode.add(Dense(2500,activation='relu'))\nmode.add(Dropout(0.3))\nmode.add(Dense(1500,activation='relu'))\nmode.add(Dropout(0.3))\nmode.add(Dense(10,activation='softmax'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mode.compile(loss = 'categorical_crossentropy'\n             ,optimizer = 'adam',metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = mode.fit(x_train,y_train,validation_data=(x_test, y_test),epochs = 50,batch_size=300,verbose = 1,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotmodelhistory(history1): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history1.history['accuracy']) \n    axs[0].plot(history1.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history1.history['loss']) \n    axs[1].plot(history1.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history1.history.keys())\n\nplotmodelhistory(history1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = mode.evaluate(x_test,y_test,verbose = 0)\nscores[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#SAVE THE MODEL\n\nsave_dir = os.path.join(os.getcwd(), 'saved_models')\nmodel_name = 'keras_cifar10_trained_model.h5'\n\n# Save model and weights\nif not os.path.isdir(save_dir):\n    os.makedirs(save_dir)\nmodel_path = os.path.join(save_dir, model_name)\nmodel.save(model_path)\nprint('Saved trained model at %s ' % model_path)\n\n# Score trained model.\nscores = model.evaluate(x_test, y_test, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}