{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.preprocessing.image import array_to_img\nimport os\nimport tensorflow as tf\nfrom keras_preprocessing.image import ImageDataGenerator\nimport random","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Image\n## First get familiar with Keras API\nKeras provides the load_img() function for loading an image from file as a PIL image object.\nimg_to_array() and array_to_img() let you convert between PIL image and numpy array"},{"metadata":{"trusted":true},"cell_type":"code","source":"# example of loading an image with the Keras API\n#from keras.preprocessing.image import load_img\n# load the image\nimg = load_img('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train/cat.1.jpg')\n# report details about the image\nprint(type(img))\nprint(img.format)\nprint(img.mode)\nprint(img.size)\n# show the image\n# img.show()\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# example of converting an image with the Keras API\n#from keras.preprocessing.image import img_to_array\n#from keras.preprocessing.image import array_to_img\n\n# convert to numpy array\nimg_array = img_to_array(img)\n# print(img_array)\nprint(img_array.dtype)\nprint(img_array.shape)\n# convert back to image\n# img_pil = array_to_img(img_array)\nprint(type(img))\n\n# the image's H:W 280:300, with RBG - 3 color channel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ok this works now Let's load in all the image"},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = os.listdir(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/train/\")\n# filenames\n# clearly the target labels are embeded in the filename, Let's parse them out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_filenames = []\ncat_filenames = []\nfor i in filenames:\n if i.split('.')[0] == 'dog':\n dog_filenames.append(i)\n else:\n cat_filenames.append(i)\n\n# At first I was using 0,1 to label target, but it throw error when using imagedatagenerator.flow_from_dataframe()\n# function -- obviously if you are using 'binary' as class_mode, the target label data type has to be string\n# Don't know why\ndog_df = pd.DataFrame({'filename':dog_filenames, 'label':'dog'})\ncat_df = pd.DataFrame({'filename':cat_filenames, 'label':'cat'})\nall_df = cat_df.append(dog_df)\n#train_data_array= train_df['filename'].apply(lambda x: load_img_array('dogs-vs-cats/train/'+ x))\n#train_label_array = train_df['label'].values\nall_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_df.label.value_counts()\n# perfectly balanced dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Scaling\n1.we need to make every pics the same size\n\n2.A best practice in neural network, to scale down the feature range for quicker convergence. \nSince this is a RBG feature, each element (pixel) is from 0 - 255, we can simply rescale by 255\n\n3.Also we need image augmentation - basically means to rotate, flip the original image to create more training images, add some spice on variation, help the model less overfit.\nAll these can be achieve by Keras ImageDataGenerator() function.\ntutorial:[https://medium.com/@vijayabhaskar96/tutorial-on-keras-flow-from-dataframe-1fd4493d237c]\nThis function also helps you batch loading the data from the directory into memory."},{"metadata":{"trusted":true},"cell_type":"code","source":"# first Let's hold out 20% of data to be our testing set\n# the images in '~/test1/' folder do not have label, we will only be apply best performance model on test dataset\n# to get optimal submission score\nfrom sklearn.model_selection import train_test_split\ntrain_df, test_df = train_test_split(all_df, test_size=0.2, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = 224 # the input shape for VGG net has to be 224 * 224, so here I set it up this way for later adoption on VGG \n\ntrain_datagen=ImageDataGenerator(\n rescale=1./255., # scaling\n rotation_range = 15, shear_range = 0.1, zoom_range = 0.1, \n horizontal_flip = True, width_shift_range=0.1,height_shift_range=0.1) # give more variants to the training dataset\n\ntrain_generator=train_datagen.flow_from_dataframe(\n dataframe=train_df,\n directory=\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/train/\", # dir path which contains all the training dataset\n x_col=\"filename\", # the col that contains the file name\n y_col=\"label\", # the col that shows the class label\n batch_size=32, # ImageDataGenerator is also a batch loading data function\n seed=42, # for reproductivbility\n shuffle=True,\n class_mode=\"binary\", \n target_size=(image_size,image_size)) # The dimensions to which all images found will be resized. in this case I am just using 150","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check out the training data shape for each batch\nfor data_batch, labels_batch in train_generator:\n print('data batch shape: ', data_batch.shape)\n print('labels batch shape: ', labels_batch.shape)\n break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# also build a test data generator for later use.\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_datagen.flow_from_dataframe(\n dataframe=test_df,\n directory='/kaggle/input/dogs-vs-cats-redux-kernels-edition/train/',\n x_col=\"filename\",\n y_col=\"label\",\n target_size=(image_size, image_size),\n batch_size=32,\n class_mode='binary')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model1 - Simple CNN Model\nJust to get ourself familiar with the idea of CNN, we are not going to throw some fancy model here.\nWe are simply using sequential model and add 3 filter & pooling layers, fully connected layer on.\nNot even tunning any data, we will do it later"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, BatchNormalization, Dropout\nfrom keras import optimizers\ninput_shape = (image_size, image_size, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# build model\nmodel = Sequential()\nmodel.add(Conv2D(32, kernel_size=(3, 3), strides=(1, 1),\n activation='relu',\n input_shape=input_shape))\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, (3,3), activation='relu'))\nmodel.add(MaxPooling2D((2,2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(1000, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid')) \nmodel.compile(optimizers.rmsprop(lr=1e-3, decay=1e-6),loss=\"binary_crossentropy\",metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Difference between fit() and fit_generator()\n\nIf your data do not fit into RAM, or if you are using image augmentation, like I do here, we should use fit_generator\nhttps://www.pyimagesearch.com/2018/12/24/how-to-use-keras-fit-and-fit_generator-a-hands-on-tutorial/\n\nHere we might have some confusion, the validation_data means -- data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. \nEarly stopping requires that a validation dataset is evaluated during training. Here I keep validation so that we can do early stopping or later we can draw the epoch versus loss line plot. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the model\nSTEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size # number of training instances // each batch size\n# in our case it is 25000 // 32 = 781\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\nmodel.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN, \n # Total number of steps (batches of samples) to yield from generator \n # before declaring one epoch finished and starting the next epoch.\n validation_data=test_generator,\n validation_steps=STEP_SIZE_TEST,\n epochs=3\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Very interestingly we can see for each epoch, the accuracy on test are higher than arrcuracy on train..wow\naccuracy on train over 3 epochs - 0.7756\naccuracy on train over 3 epochs - 0.7903"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"./model1_simpleCNN.hdf5\")\n# we only got accuracy of 0.7175 not great, but we have a base line model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# just to download the output model file from kaggle to laptop\nfrom IPython.display import FileLink\nFileLink(r'model1_simpleCNN.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model2 - Simple CNN but with tunned hyperparameters \nLet's try use randomsearch to tunned hyperparameters\nIn general you want to train for learning_rate first, and I will also try add dropout / batchnormalization / optimization in this one. Just for simplicity using only 3 epochs."},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\ndef create_CNN(activation = 'relu', dropout_flag = False, batchnorm_flag = False, \n dropout_rate = 0.5, learning_rate = 1e-4, optimizer = 'rmsprop'):\n model = Sequential()\n model.add(Conv2D(32, kernel_size=(3, 3), strides=(1, 1),\n activation = activation,\n input_shape=input_shape))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Conv2D(64, (3, 3), activation=activation))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2, 2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Conv2D(128, (3,3), activation=activation))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2,2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Flatten())\n model.add(Dense(1000, activation=activation))\n model.add(Dense(1, activation='sigmoid')) \n if optimizer == 'sgd':\n optimize_instance = keras.optimizers.SGD(lr = learning_rate)\n elif optimizer == 'adam':\n optimize_instance = keras.optimizers.Adam(learning_rate = learning_rate, beta_1=0.9, beta_2=0.999, amsgrad=False)\n elif optimizer == 'rmsprop':\n optimize_instance = keras.optimizers.RMSprop(learning_rate= learning_rate, rho=0.9, decay = 1e-6)\n elif optimizer == 'adagrad':\n optimize_instance = keras.optimizers.Adagrad(learning_rate= learning_rate)\n else:\n print('Not a valid optimizer') \n model.compile(optimize_instance,loss=\"binary_crossentropy\",metrics=[\"accuracy\"])\n return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"activations = ['relu', 'tanh', 'sigmoid', 'linear']\noptimizers = ['adam', 'rmsprop', 'adagrad']\n# I cross out SGD optimizer Because the accuracy over 3 epochs are around 57%, barely smarter than a ramdom guess..\n# \nlearning_rates = [1e-5, 1e-4,1e-3]\ndropoutrates = [0.5, 0.25]\ninitializers = ['glorot_uniform','glorot_normal','he_normal', 'he_uniform']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Because I didn't find gridsearchCV and randomsearchCV that can be used with fit_generator, I will have to define my own sudo-randomsearch. Not going to do any cross validation that would be too expensive, but just randomly sample elements from the param_gird and "},{"metadata":{"trusted":true},"cell_type":"code","source":"def sample_params(param_grid, n): # n = how many sample combination you want\n l = []\n num = 0\n for i in range(n):\n d = {}\n for k, v in param_grid.items():\n d[k] = random.sample(v,1)\n l.append(d)\n return l","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"However, trying too many combinations might explode kaggle kernel. I will instead do a gridsearch on optimizer and learning_rate first. And also this time we will be train on a CNN that has dropout and batch normalization layers\nThis time we will be using 3 epoch and looking at the validation accuracy to decide further hyperparameter tunning."},{"metadata":{"trusted":true},"cell_type":"code","source":"param_grid = dict(\n# activation = activations, \n optimizer = optimizers, \n learning_rate = learning_rates,\n batchnorm_flag = [True],\n dropout_flag = [True],\n# dropout_rate = dropoutrates\n \n)\nimport itertools as it\nparam_combinations = [dict(zip(param_grid,v)) for v in it.product(*param_grid.values())]\n#len(param_combinations) # 9 combinations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size # number of training instances // each batch size\n# in our case it is 25000 // 32 = 781\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\n\n# iterate over each combination of hyperparameters\ncount = 1\nfor i in param_combinations:\n print('--------train CNN ',count, ' -----------')\n print('HyperParameters: ', i)\n model = create_CNN(**i)\n model.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN, \n # Total number of steps (batches of samples) to yield from generator \n # before declaring one epoch finished and starting the next epoch.\n validation_data=test_generator,\n validation_steps=STEP_SIZE_TEST,\n epochs=3\n )\n count +=1\n# Now you just wait and see..","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"(accuracy on train, accuracy on test) over 3 epochs\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
optimizerlr = 1e-05lr = 1e-04lr = 1e-03
adam0.6463, 0.56840.7219, 0.71520.6641, 0.5775
rmsprop0.6485, 0.52720.6955, 0.68600.7447, 0.7160
adagrad0.5757, 0.53000.6266, 0.58940.6641, 0.4996
\n\nLearning rate is a big factor, we can see that as learning_rate 0.001 and 0.00001 work better for all 3 optimizers.\nHowever, the best optimizer is still the rmsprop. Eventhough we only have 3 epochs we can see that lr=0.001&rmsprop has the highest accuracy rate on train among all - 0.7447. The corresponding test rate are 0.7160, which is a good sign, the gap between the accuracy on test & loss is not big we can increase epoch to get better results."},{"metadata":{},"cell_type":"markdown","source":"# Model3 - fine tune model 1, add 1 fc layer\nHowever, none of these combination in model2 section are better than model1(model without dropout and batchnormalization layer). Ok fine. Let‘s went back to model1, this time we add 1 dense layer and dropout layer.\nWe also test different weight initializer and dropout rate."},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\ndef create_CNN3(activation = 'relu', dropout_flag = False, batchnorm_flag = False, \n dropout_rate = 0.5, learning_rate = 1e-4, optimizer = 'rmsprop', initializer = 'glorot_uniform'):\n model = Sequential()\n model.add(Conv2D(32, kernel_size=(3, 3), strides=(1, 1),\n activation = activation,\n input_shape=input_shape,\n kernel_initializer = initializer))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Conv2D(64, (3, 3), activation=activation, kernel_initializer = initializer))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2, 2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Conv2D(128, (3,3), activation=activation, kernel_initializer = initializer))\n if batchnorm_flag:\n model.add(BatchNormalization())\n model.add(MaxPooling2D(pool_size=(2,2)))\n if dropout_flag:\n model.add(Dropout(dropout_rate))\n\n model.add(Flatten())\n model.add(Dense(1000, activation=activation, kernel_initializer = initializer))\n # add drop out layer and fc layer\n model.add(Dropout(dropout_rate))\n model.add(Dense(1000, activation = activation, kernel_initializer = initializer))\n \n model.add(Dense(1, activation='sigmoid')) \n if optimizer == 'sgd':\n optimize_instance = keras.optimizers.SGD(lr = learning_rate)\n elif optimizer == 'adam':\n optimize_instance = keras.optimizers.Adam(learning_rate = learning_rate, beta_1=0.9, beta_2=0.999, amsgrad=False)\n elif optimizer == 'rmsprop':\n optimize_instance = keras.optimizers.RMSprop(learning_rate= learning_rate, rho=0.9, decay = 1e-6)\n elif optimizer == 'adagrad':\n optimize_instance = keras.optimizers.Adagrad(learning_rate= learning_rate)\n else:\n print('Not a valid optimizer') \n model.compile(optimize_instance,loss=\"binary_crossentropy\",metrics=[\"accuracy\"])\n return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_grid = dict(\n# activation = activations, \n# optimizer = optimizers, \n# learning_rate = learning_rates,\n# batchnorm_flag = [True],\n# dropout_flag = [True],\n dropout_rate = dropoutrates,\n initializer = initializers\n \n)\nimport itertools as it\nparam_combinations = [dict(zip(param_grid,v)) for v in it.product(*param_grid.values())]\n#len(param_combinations) # 9 combinations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size # number of training instances // each batch size\n# in our case it is 25000 // 32 = 781\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\n\n# iterate over each combination of hyperparameters\ncount = 1\nfor i in param_combinations:\n if count != 1 and count != 2:\n print('--------train CNN ',count, ' -----------')\n print('HyperParameters: ', i)\n model = create_CNN3(**i)\n model.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN, \n # Total number of steps (batches of samples) to yield from generator \n # before declaring one epoch finished and starting the next epoch.\n validation_data=test_generator,\n validation_steps=STEP_SIZE_TEST,\n epochs=3\n )\n count +=1\n# Now you just wait and see..","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
initializerdropoutrateaccuracy on trainaccuracy on test
glorot_uniform0.50.75560.7772
glorot_normal0.50.74560.7760
he_normal0.50.74560.7760
he_uniform0.50.75250.7513
glorot_uniform0.250.77160.7919
glorot_normal0.250.76480.7903
he_normal0.250.75390.7818
he_uniform0.250.74880.7649
"},{"metadata":{},"cell_type":"markdown","source":"As we can see that glorot_uniform with 0.25 dropout rate performs the best. Let's train the model over 80 epochs.\n\nTo make sure the training process could be stopped early if the loss on validation set did not decrease, we will use callback to early stop the training process."},{"metadata":{"trusted":true},"cell_type":"code","source":"m3 = create_CNN3(dropout_rate = 0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping\nes = EarlyStopping(monitor = 'val_accuracy', mode = 'max', verbose = 1, patience = 3)\n# patience = 3 means that if the accuracy on test set does not improve over 3 epochs we will stop the training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the model\nSTEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size # number of training instances // each batch size\n# in our case it is 25000 // 32 = 781\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\nhistory = m3.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN, \n # Total number of steps (batches of samples) to yield from generator \n # before declaring one epoch finished and starting the next epoch.\n validation_data=test_generator,\n validation_steps=STEP_SIZE_TEST,\n epochs=80,\n callbacks=[es]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot traning histoy -- loss\nplt.plot(history.history['loss'], label = 'train')\nplt.plot(history.history['val_loss'], label = 'test')\nplt.title('loss')\nplt.xlabel('epochs')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot traning histoy\nplt.plot(history.history['accuracy'], label = 'train')\nplt.plot(history.history['val_accuracy'], label = 'test')\nplt.title('accuracy')\nplt.xlabel('epochs')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"okay, not that exhilarating, I plan to train on 80 epochs, however it stopped at the 12th epoch, reaching training accuracy of 0.8372, and test accuracy of 0.8533\n\nVery strangly, the accuracy on train is always less than that on test ,,, I guess it might due to the way I generate the train and test data. I apply image augmentation on the train set but not on the test, making training data much trickier to be dealt with."},{"metadata":{"trusted":true},"cell_type":"code","source":"# save model\nm3.save(\"./model3_tunnedCNN.hdf5\")\n# to download the output model file from kaggle to laptop\nfrom IPython.display import FileLink\nFileLink(r'model3_tunnedCNN.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model4 - transfer learning, base model as feature extractor\n"},{"metadata":{},"cell_type":"markdown","source":"There is an old saying in Chinese that 'we should all learn from the wisdom of the elder'. In the deep learning context, we could say we should learn from the pre-trained model. \n\nThe author of the third link shown below, utilized InceptionV3 as the base model and achieved 94% accuracy on cat vs. dog classification. So I will re-implement his solution with a little bit twist on the input data augmentation and training with earlystop.\n\nGreat articles articulate transfer learning\n\n1.https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html\n\n2.https://codelabs.developers.google.com/codelabs/keras-flowers-transfer-learning/#3\n\n3.https://medium.com/abraia/first-steps-with-transfer-learning-for-custom-image-classification-with-keras-b941601fcad5"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.inception_v3 import InceptionV3\nbase_model = InceptionV3(weights='imagenet', include_top=False) \n# we don't need the final output layer, we just need the feature activation map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# freeze each layer of base model, which means that we are not going to update weights or biases along the training process\nfor layer in base_model.layers:\n layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import GlobalAveragePooling2D, Dropout, Dense\nfrom keras.models import Model\n# keras model API\nx = base_model.output\nx = GlobalAveragePooling2D(name='avg_pool')(x) # you don't have to name the layer\nx = Dropout(0.25)(x) # based on the tunned hyperparamters of model 3\nx = Dense(256, activation = 'relu')(x)\npredictions = Dense(1, activation='sigmoid')(x)\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\nmodel.compile(optimizer='rmsprop',\n loss='binary_crossentropy',\n metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we use GlobalAveragePooling before FC layer, (I don't know if this GAP layer would improve performance, you can try train one without GAP layer). Anyway, on top of the ability to indicate what is the object in the image, one cool effect of GAP layer is that it can help us identify where the object is in the image, which is called object localization. More info you can refer to the following link\nhttps://alexisbcook.github.io/2017/global-average-pooling-layers-for-object-localization/"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping\nes = EarlyStopping(monitor = 'val_accuracy', mode = 'max', verbose = 1, patience = 3)\n# patience = 3 means that if the accuracy on test set does not improve over 3 epochs we will stop the training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the model\nSTEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size # number of training instances // each batch size\n# in our case it is 25000 // 32 = 781\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\nhistory = model.fit_generator(generator=train_generator,\n steps_per_epoch=STEP_SIZE_TRAIN, \n # Total number of steps (batches of samples) to yield from generator \n # before declaring one epoch finished and starting the next epoch.\n validation_data=test_generator,\n validation_steps=STEP_SIZE_TEST,\n epochs=80,\n callbacks=[es]\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1Remarkable, after the fist epoch. the model achieved 0.8867 accuracy on train and 0.9866 on test\nAs epoch increases, the accuracy on test dataset go up gradually to 0.94 ， accuracy on test remains relatively stable at around 0.9865. \nA complete slam dunk over the previous basic CNN models..."},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot traning histoy -- loss\nplt.plot(history.history['loss'], label = 'train')\nplt.plot(history.history['val_loss'], label = 'test')\nplt.title('model4 - transfered model - loss')\nplt.xlabel('epochs')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot traning histoy\nplt.plot(history.history['accuracy'], label = 'train')\nplt.plot(history.history['val_accuracy'], label = 'test')\nplt.title('model4 - transfered model - accuracy')\nplt.xlabel('epochs')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('./model4_transfer_lr_feature_extractor.hdf5')\n# to download the output model file from kaggle to laptop\nfrom IPython.display import FileLink\nFileLink(r'model4_transfer_lr_feature_extractor.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict on test\nuse model4 - the transfered learning model"},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = os.listdir(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/test/\")\npredictset = pd.DataFrame({'filename': filenames})\npredictset.sort_values(by='filename', inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# what is the last row.by index..delete that \npredictset.drop(4505, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictIterator = test_datagen.flow_from_dataframe(dataframe = predictset,\n directory = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test/',\n x_col = 'filename', y_col = None,\n target_size = (image_size, image_size),\n batch_size = 32,\n class_mode = None,\n shuffle = False) # don't shuffle, we are here to predict each image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = model.predict_generator(predictIterator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.DataFrame({})\nsubmit['id'] = predictset.filename.str.split('.').str[0]\nsubmit['label'] = np.round(labels[:,0]).astype(int)\nsubmit.to_csv('submission_pa_hw6.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}