{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#This kernel aims to investigate the performance of trasfer learning in solving the Dogs vs. Cats image recognition problem\n#In the previous kernel we have seen that the amount of data available during training phase is substantial to the performance of model applying to unknown data\n#In the following we are going to use pretrained convolutional base from the VGG16 model which is trained with a huge dataset in ImageNet\n#and see if this can achieve a better performance than the model in the previous kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%xmode Plain\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns;sns.set()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense,Conv2D,MaxPooling2D,Flatten,Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import models\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.applications import VGG16\nfrom tensorflow.keras import backend as Back","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def result_visualisation(loss,acc,val_loss,val_acc):\n    #Result visualisation\n    epochs = np.arange(1,len(loss)+1)\n    fig,ax = plt.subplots(1,2,figsize=(20,5))\n    ax[0].plot(epochs,loss,label='loss')\n    ax[0].plot(epochs,val_loss,label='val_loss')\n    ax[0].set_title('Loss')\n    ax[0].set_xlabel('Epochs')\n    ax[0].legend(['loss','val_loss'])\n    ax[1].plot(epochs,acc,label='acc')\n    ax[1].plot(epochs,val_acc,label='val_acc')\n    ax[1].set_title('Accuracy')\n    ax[1].set_xlabel('Epochs')\n    ax[1].legend(['acc','val_acc'])\n    plt.tight_layout()\n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Directories of data\ntrain_dir = '../input/dogs-vs-cats/train/train'\ntest_dir = '../input/dogs-vs-cats/test1/test1'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create labels for the images\nfilenames = os.listdir(train_dir)\nlabels = []\nfor filename in filenames:\n    if filename.startswith('dog'):\n        labels.append(1)\n    else:\n        labels.append(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import the filenames and labels into a dataframe\n#Only 5000 images would be used for a shorter training time\ndf_train = pd.DataFrame({'Filenames':filenames,'Labels':labels})\n#Stratified sampling\ndf_train = df_train.groupby('Labels').apply(lambda x:x.sample(frac=0.2,random_state=100))\n#Read data from the directory\nX_filenames = df_train['Filenames'].values\ny = df_train['Labels'].values\nX = [image.load_img(os.path.join(train_dir,filename),target_size=(150,150)) for filename in X_filenames]\nX = np.array([image.img_to_array(item) for item in X])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Standardization of training data\nX = X/255.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Split the data into train and val set\nXtrain,Xval,ytrain,yval = train_test_split(X,y,stratify=y,test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Data Augmentation\nimggen = ImageDataGenerator(rotation_range=30,width_shift_range=0.2,height_shift_range=0.2,\n                            shear_range=0.2,horizontal_flip=True,fill_mode='nearest')\ntrain_gen = imggen.flow(Xtrain,ytrain,batch_size=128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Extract convolutional base from VGG16 and construct the network\n#As this kernel is applying to a binary classification problem, so we are going to use only the convolutional base from the VGG16 model\n#A new fully connected layer will be constructed and trained\n#Convolutional base will be freezed and only the newly created fully connected layer will be trained in this phase\nconv_base = VGG16(weights='imagenet',include_top=False,input_shape=(150,150,3))\nconv_base.trainable = False\nCNN = Sequential()\nCNN.add(conv_base)\nCNN.add(Flatten())\nCNN.add(Dropout(0.5))\nCNN.add(Dense(512,activation='relu'))\nCNN.add(Dense(1,activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the optimizer and compile the CNN\noptimizer = RMSprop(lr=1e-4)\nCNN.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Examine the structure of the CNN model\nCNN.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Training\nhistory1 = CNN.fit_generator(train_gen,steps_per_epoch=80,validation_data=(Xval,yval),epochs=80,verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Extract loss and acc for visualisation\nloss1 = history1.history['loss']\nacc1 = history1.history['acc']\nval_loss1= history1.history['val_loss']\nval_acc1 = history1.history['val_acc']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Result visualisation\nresult_visualisation(loss1,acc1,val_loss1,val_acc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the results above, we can see that they are better than those produced by a CNN network trained from the beginning\n#The main reason of having such great improvement would be due to the availability of training dataset during the training phase of the convolutional base\n#Another interesting observation here is the validation accuracy is better than the training accuracy\n#This is mainly due to the existence of the dropout layer, during training phase not all activation nodes are available for prediction, while they are all available during\n#the validation phase, so the performance on validation data is better than on training data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Examine the activation outputs of the convolutional base\nlayer_names = ['block1_conv2','block3_conv2','block5_conv2']\nimg = Xtrain[0].reshape(1,150,150,3)\noutputs = [conv_base.layers[index].output for index in [2,8,16]]\nactivation_model = models.Model(inputs=conv_base.input,outputs=outputs)\nactivation_outputs = activation_model.predict(img)\nfor i,output in enumerate(activation_outputs):\n    plt.figure(i,figsize=(20,20))\n    for j in range(1,4):\n        plt.subplot(1,4,j)\n        plt.imshow(output[0,:,:,j],cmap='viridis')\n        plt.title(layer_names[i]+' activation output {}'.format(j))\n        plt.grid(False)\n        plt.axis('off');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualise the convolutional filters\nfilter_num = 25\nlearning_step = 1.\nfor i,layer_name in enumerate(layer_names):\n    plt.figure(i,figsize=(20,20))\n    for j in range(filter_num):\n        layer_output = conv_base.get_layer(layer_name).output\n        loss = Back.mean(layer_output[:,:,:,j])\n        grad = Back.gradients(loss,conv_base.input)[0]\n        grad = grad/Back.sqrt(Back.mean(Back.square(grad))+1e-5)\n        func = Back.function([conv_base.input],[loss,grad])\n        input_img = np.random.random((1,150,150,3))*10+128.\n        #Using gradient ascent to maximize the loss\n        for k in range(30):\n            loss_value,grad_value = func([input_img])\n            input_img += grad_value*learning_step\n        #Standardize the image data\n        input_img -= np.mean(input_img)\n        input_img /= np.std(input_img)\n        input_img += 0.5\n        input_img = np.clip(input_img,0,1)\n        #Change to RGB format\n        input_img *= 255.\n        input_img = np.clip(input_img,0,255).astype('uint8')\n        #Visualise the filter output\n        plt.subplot(5,5,j+1)\n        plt.imshow(input_img[0])\n        plt.title(layer_name + ' filter {}'.format(j))\n        plt.grid(False)\n        plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the visualisation of filers observed above, we can see that the deeper the layer, the more abstract and complicated the filter is\n#This suggests that the filters in the shallower layers are of more general purpose, such as edge detection, so they are more suitable to be used in transfer learning\n#The deeper layers are of less general function, and this suggests that we may be able to train them in order to achieve better results for specific problems","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#In the following, we are going to investigate whether we can further improve the performance of this model by fine-tuning part of the convolutional base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Examine the structure of the convolutional base\nconv_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Unfreeze part of the convoluational base\nlayer_unfreeze = ['block5_conv1','block5_conv2','block5_conv3']\nconv_base.trainable = True\nfor layer in conv_base.layers:\n    if layer.name in layer_unfreeze:\n        layer.trainable = True\n    else:\n        layer.trainable = False\nCNN.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CNN.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Training\nhistory2 = CNN.fit_generator(train_gen,steps_per_epoch=80,validation_data=(Xval,yval),epochs=25,verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Extract loss and acc for visualisation\nloss2 = history2.history['loss']\nacc2 = history2.history['acc']\nval_loss2 = history2.history['val_loss']\nval_acc2 = history2.history['val_acc']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Result visualisation\nresult_visualisation(loss2,acc2,val_loss2,val_acc2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the results above, we can observe that unfreezing part of the convolutional base and fine tuning the weights would improve the performance of  the network","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}