{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1.Import Library & DataSet","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Import Tensorflow & Keras\nfrom tensorflow import keras\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.regularizers import l2\nfrom keras.utils import conv_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.engine.topology import get_source_inputs\nfrom keras.applications.resnet50 import preprocess_input\nfrom keras.applications.imagenet_utils import decode_predictions\nfrom keras import backend as K\nfrom keras.callbacks import EarlyStopping\nfrom keras.utils import to_categorical\n\nimport tensorflow as tf\n\n# Helper libraries\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Import DataSets\nfrom keras.datasets import cifar10\n(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dict\ndictionary={1:\"automobile\",2:\"bird\",3:\"cat\",4:\"deer\",5:\"dog\",6:\"frog\",7:\"horse\",8:\"ship\",9:\"truck\",0:\"airplane\"}\n# visualizing training samples\nplt.figure(figsize=(15,5))\nfor i in range(40):  \n    plt.subplot(4, 10, i+1)\n    plt.imshow(train_images[i].reshape((32, 32, 3)),cmap=plt.cm.hsv)\n    plt.title(dictionary[train_labels[i][0]])\n    plt.axis('off')\nplt.subplots_adjust(wspace=0.3, hspace=0.3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data Preprocess","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize pixel values to be between 0 and 1\ntrain_images.astype('float32');test_images.astype('float32')\ntrain_images, test_images = train_images / 255.0, test_images / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encoding\ntrain_labels = to_categorical(np.array(train_labels[:, 0]))\ntest_labels = to_categorical(np.array(test_labels[:, 0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3.Model Design\n## 3.1 Squeeze and Excitation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def squeeze_excite_block(input, ratio=16):\n    \n    init = input\n    filters = init._keras_shape[-1]\n    se_shape = (1, 1, filters)\n\n    se = GlobalAveragePooling2D()(init)\n    se = Reshape(se_shape)(se)\n    se = Dense(filters // ratio, activation='relu', use_bias=False)(se)\n    se = Dense(filters, activation='sigmoid', use_bias=False)(se)\n    x = multiply([init, se])\n    \n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.2 ResNet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def resnet_block(input, filters, k=1, strides=(1, 1)):\n    init = input\n    channel_axis = -1 \n\n    x = BatchNormalization(axis=channel_axis)(input)\n    x = Activation('relu')(x)\n\n    if strides != (1, 1) or init._keras_shape[channel_axis] != filters * k:\n        init = Conv2D(filters * k, (1, 1), padding='same', kernel_initializer='he_normal',\n                      use_bias=False, strides=strides)(x)\n\n    x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal',\n               use_bias=False, strides=strides)(x)\n    x = BatchNormalization(axis=channel_axis)(x)\n    x = Activation('relu')(x)\n\n    x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal',\n               use_bias=False)(x)\n\n    # squeeze and excite block\n    x = squeeze_excite_block(x)\n\n    m = add([x, init])\n    return m","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.3 SE_ResNet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_se_resnet(classes, img_input, include_top, initial_conv_filters, filters,\n                      depth, width, weight_decay, pooling):\n\n    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n    N = list(depth)\n\n    x = Conv2D(initial_conv_filters, (7, 7), padding='same', use_bias=False, strides=(2, 2),\n               kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(img_input)\n\n    x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)\n\n    for i in range(N[0]):\n            x = resnet_block(x, filters[0], width)\n\n    for k in range(1, len(N)):\n            x = resnet_block(x, filters[k], width, strides=(2, 2))\n\n    for i in range(N[k] - 1):\n            x = resnet_block(x, filters[k], width)\n\n    x = BatchNormalization(axis=channel_axis)(x)\n    x = Activation('relu')(x)\n\n\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(classes, use_bias=False, kernel_regularizer=l2(weight_decay),\n                  activation='softmax')(x)\n    \n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4.Model Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_rows, img_cols = 32, 32\ninput_shape = (img_rows, img_cols, 3)\ninitial_conv_filters=64\ndepth=[2, 2, 2, 2]\nfilters=[64, 128, 256, 512]\nwidth=1\nweight_decay=1e-4\ninclude_top=True\nweights=None\ninput_tensor=None\npooling=None\nclasses=10\n\nimg_input = Input(shape=input_shape)\n   \nx = create_se_resnet(classes, img_input, include_top, initial_conv_filters,\n                          filters, depth, width, weight_decay, pooling)\n\nmodel = Model(img_input, x, name='resnext')\nprint('model created')\n    \nmodel.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_images, train_labels, epochs=20, validation_split=0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn')\nplt.figure(figsize = (16,8))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Training Result',fontsize=20)\nplt.ylabel('Loss',fontsize=16)\nplt.xlabel('Epoch',fontsize=16)\nplt.legend(['accuracy','Validation_accuracy'], loc='lower right',fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(test_images, test_labels, verbose=0)\nprint('accuracy: ',score[1])\nprint('loss: ',score[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5.Bag of tricks\n## 5.1 Nesterov Accelerated Gradient","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')\nmodel.compile(optimizer='Nadam',loss='categorical_crossentropy',metrics=['accuracy'])\nhistory_Nadam = model.fit(train_images, train_labels, epochs=20, validation_split=0.25,callbacks=[earlyStopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn')\nplt.figure(figsize = (16,8))\nplt.plot(history_Nadam.history['accuracy'])\nplt.plot(history_Nadam.history['val_accuracy'])\nplt.title('Training Result',fontsize=20)\nplt.ylabel('Loss',fontsize=16)\nplt.xlabel('Epoch',fontsize=16)\nplt.legend(['accuracy','Validation_accuracy'], loc='lower right',fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(test_images, test_labels, verbose=0)\nprint('accuracy: ',score[1])\nprint('loss: ',score[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5.2 Cosine Learning Rate Decay","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nfrom keras.callbacks import Callback\nfrom keras import backend as K\n\nclass CosineAnnealingScheduler(Callback):\n\n    def __init__(self, T_max, eta_max, eta_min=0, verbose=0):\n        super(CosineAnnealingScheduler, self).__init__()\n        self.T_max = T_max\n        self.eta_max = eta_max\n        self.eta_min = eta_min\n        self.verbose = verbose\n\n    def on_epoch_begin(self, epoch, logs=None):\n        if not hasattr(self.model.optimizer, 'lr'):\n            raise ValueError('Optimizer must have a \"lr\" attribute.')\n        lr = self.eta_min + (self.eta_max - self.eta_min) * (1 + math.cos(math.pi * epoch / self.T_max)) / 2\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nEpoch %05d: CosineAnnealingScheduler setting learning '\n                  'rate to %s.' % (epoch + 1, lr))\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [CosineAnnealingScheduler(T_max=100, eta_max=1e-2, eta_min=1e-4)]\nmodel.compile(optimizer='Nadam',loss='categorical_crossentropy',metrics=['accuracy'])\nhistory_Nadam_cosLR = model.fit(train_images, train_labels, epochs=20, validation_split=0.25,callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn')\nplt.figure(figsize = (16,8))\nplt.plot(history_Nadam_cosLR.history['accuracy'])\nplt.plot(history_Nadam_cosLR.history['val_accuracy'])\nplt.title('Training Result',fontsize=20)\nplt.ylabel('Loss',fontsize=16)\nplt.xlabel('Epoch',fontsize=16)\nplt.legend(['accuracy','Validation_accuracy'], loc='lower right',fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(test_images, test_labels, verbose=0)\nprint('accuracy: ',score[1])\nprint('loss: ',score[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After using cosine learning rate decay & Nesterov Accelerated Gradient as optimizer, judging from the verification accuracy during the training process, we found that the validation accuracy increased more steadily than traditional learning rate.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 6.Predict Result Review","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plot\nimport math\nlabel_desc = [ 'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck' ]\ndef show_feature_label_prediction( features\n                                 , labels\n                                 , predictions\n                                 , indexList\n                                 ) :\n    num = len(indexList)\n    plot.gcf().set_size_inches( 2*5, (2+0.4)*math.ceil(num/5) )\n    loc = 0\n    for i in indexList :\n        loc += 1\n        subp = plot.subplot( math.ceil(num/5), 5, loc )\n        subp.imshow( features[i], cmap='binary' )\n        if( len(predictions) > 0 ) :\n            title = 'ai = ' + label_desc[ predictions[i] ]\n            title += (' (o)' if predictions[i]==labels[i] else ' (x)')\n            title += '\\nlabel = ' + label_desc[ labels[i] ]\n        else :\n            title = 'label = ' + label_desc[ labels[i] ]\n        subp.set_title( title, fontsize=12 )\n        subp.set_xticks( [] )\n        subp.set_yticks( [] )\n    plot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(train_images2, train_labels2), (test_images2, test_labels2) = cifar10.load_data()\npredict = model.predict(test_images)\npredict=np.argmax(predict,axis=1)\ntest_label_onearr = test_labels2.reshape(len(test_labels2))\nshow_feature_label_prediction(test_images, test_label_onearr, predict, range(0, 10) )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Top-20 predict error list","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"checkList = pd.DataFrame( {'label':test_label_onearr,'prediction':predict})\nshow_feature_label_prediction(test_images, test_label_onearr, predict, checkList.index[checkList.prediction != checkList.label][0:20])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}