{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pickle\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport pandas as pd\n\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom lrutils import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"IMG_SHAPE = (128, 128, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load the data\n\nIn previous work I have already divided the Dogs vs. Cats dataset into training (9/10ths) and validation (1/10th) sets, randomly selected from the full training data. Each contains a 50/50 mix of dog and cat images."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Load data\ntrain_df = pd.read_pickle('../input/cat-vs-dog-data/train_df.pkl')\ntrain_df.filename = train_df.filename.map(lambda s: s.split('\\\\')[1])\nvalidate_df = pd.read_pickle('../input/cat-vs-dog-data/validate_df.pkl')\nvalidate_df.filename = validate_df.filename.map(lambda s: s.split('\\\\')[1])\ntrain_dir = '../input/dogs-vs-cats/train/train'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"During training, apply various random transformations to the images, as a form of dataset augmentation."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   rotation_range=20,\n                                   width_shift_range=0.2,\n                                   height_shift_range=0.2,\n                                   shear_range=0.2,\n                                   zoom_range=0.2,\n                                   horizontal_flip=True)\nvalidation_datagen = ImageDataGenerator(rescale = 1./255.)\n\ntrain_generator = train_datagen.flow_from_dataframe(train_df,\n                                                    train_dir,\n                                                    x_col='filename',\n                                                    y_col='category',\n                                                    batch_size=50,\n                                                    class_mode='binary',\n                                                    target_size=(128,128))     \nvalidation_generator =  validation_datagen.flow_from_dataframe(validate_df,\n                                                          train_dir,\n                                                          x_col='filename',\n                                                          y_col='category',\n                                                         batch_size=50,\n                                                         class_mode  = 'binary',\n                                                         target_size = (128,128))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utility methods"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    #-----------------------------------------------------------\n    # Retrieve a list of list results on training and test data\n    # sets for each training epoch\n    #-----------------------------------------------------------\n    acc      = history.history[     'acc' ]\n    val_acc  = history.history[ 'val_acc' ]\n    loss     = history.history[    'loss' ]\n    val_loss = history.history['val_loss' ]\n\n    epochs   = range(len(acc)) # Get number of epochs\n\n    #------------------------------------------------\n    # Plot training and validation accuracy per epoch\n    #------------------------------------------------\n    plt.plot  ( epochs,     acc , label='Training')\n    plt.plot  ( epochs, val_acc , label='Validation')\n    plt.xlabel ('Epoch')\n    plt.ylabel ('Accuracy')\n    plt.legend ()\n    plt.title ('Training and validation accuracy')\n    plt.figure()\n\n    #------------------------------------------------\n    # Plot training and validation loss per epoch\n    #------------------------------------------------\n    plt.plot  ( epochs,     loss, label='Training')\n    plt.plot  ( epochs, val_loss, label='Validation')\n    plt.xlabel ('Epoch')\n    plt.ylabel ('Loss')\n    plt.legend ()\n    plt.title ('Training and validation loss'   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def report(model, history=None, validation_generator=None):\n    if history is not None:\n        plot_history(history)\n    \n    if validation_generator is not None:\n        # Evaluate trained model on validation set\n        validation_generator.reset()\n        [val_loss, val_acc] = model.evaluate_generator(validation_generator)\n        print('Model evaluation')\n        print(f'val_loss: {val_loss:.4f}, val_acc: {val_acc:.4f}')\n        print()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pre-trained image feature extractor\n\nKeras comes packaged with many popular models.\n\nhttps://keras.io/applications\n\nBy specifying the `include_top=False` argument, we load just the bottom convolutional layers, and not the classification layer(s) at the top."},{"metadata":{"trusted":true},"cell_type":"code","source":"mobilenet = keras.applications.MobileNetV2(input_shape = IMG_SHAPE,\n                                                   include_top = False,\n                                                   weights = 'imagenet')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Freeze feature extractor params for now."},{"metadata":{"trusted":true},"cell_type":"code","source":"mobilenet.trainable = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is a complex model with a large number of filters in the final layer. I will use the activations of those filters as the starting point for my classifier, which will be a dense neural network (with batch normalization and dropout)."},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet = keras.Sequential([\n    mobilenet,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(1, activation='sigmoid')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The classifier has this architecture for a few reasons:\n1. Large enough number of parameters so that tuning the learning rate becomes interesting.\n2. Few enough parameters that it trains in decent time on my laptop.\n3. Layer widths go down by factors of two, following general convention.\n4. The lowest layer has close to the number of units as the final number of filters in the feature extractor, which gives the model a chance to reorganize the filter outputs without losing much information.\n5. It has about as many parameters as the feature extractor, and symmetry is nice."},{"metadata":{},"cell_type":"markdown","source":"## Test an initial learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_learning_rate = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.compile(optimizer=keras.optimizers.RMSprop(learning_rate=base_learning_rate),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"savebest = keras.callbacks.ModelCheckpoint('mobilenet_weights.h5', monitor='val_loss', mode='min',\n                                             save_best_only=True, save_weights_only=True,\n                                             verbose=1, save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = classifier_mobilenet.fit_generator(train_generator,\n                              epochs=10,\n                              validation_data=validation_generator,\n                                callbacks=[savebest])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.load_weights('mobilenet_weights.h5')\nreport(classifier_mobilenet, history, validation_generator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tuning the learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet = keras.Sequential([\n    mobilenet,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(1, activation='sigmoid')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.compile(optimizer=keras.optimizers.RMSprop(),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Start with a learning rate of 1e-6, and multiply this by sqrt(2) every batch."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_schedule = keras.callbacks.LearningRateScheduler(lambda epoch: 1e-6 * 2**(epoch/2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = classifier_mobilenet.fit_generator(train_generator,\n                              epochs=30,\n                              validation_data=validation_generator,\n                                callbacks=[lr_schedule],\n                                steps_per_epoch=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the tuning history\nlrs = 1e-6 * 2**(np.arange(30)/2)\nplt.semilogx(lrs, history.history[\"val_loss\"])\nplt.axis([1e-5, 1e-2, 0.2, 1.0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(20,25):\n    print(f'lr = {history.history[\"lr\"][epoch]}, val_loss: {history.history[\"val_loss\"][epoch]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Learning starts to become unstable for lr above about 0.0014, so I will consider this the highest \"good\" learning rate."},{"metadata":{"trusted":true},"cell_type":"code","source":"tuned_learning_rate = 0.0014","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model with tuned learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet = keras.Sequential([\n    mobilenet,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(1, activation='sigmoid')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.compile(optimizer=keras.optimizers.RMSprop(learning_rate=tuned_learning_rate),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"savebest = keras.callbacks.ModelCheckpoint('tunedlr_weights.h5', monitor='val_loss', mode='min',\n                                             save_best_only=True, save_weights_only=True,\n                                             verbose=1, save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = classifier_mobilenet.fit_generator(train_generator,\n                              epochs=10,\n                              validation_data=validation_generator,\n                                callbacks=[savebest])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.load_weights('tunedlr_weights.h5')\nreport(classifier_mobilenet, history, validation_generator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**NOTE:** There is a notable degree of random variation if the model is independently trained multiple times - I have seen the validation set accuracy vary by roughly three quarters of a percent for the models in this notebook. A model that does better after training with one strategy may not always do better than if it were trained with another strategy. For a more rigorous comparison of different training strategies, multiple training runs are essential."},{"metadata":{},"cell_type":"markdown","source":"## Cyclical learning rates\n\nIn this strategy, the learning rate follows (the first half of) a cosine curve over the course of an epoch, and then resets and starts the cycle again for the next epoch, etc.\n\nOriginal code for `LR_Updater` and `LR_Cycle` from this repo:<br>\nhttps://github.com/gunchagarg/learning-rate-techniques-keras\n\nClarified and condensed code for `LR_Updater` and `LR_Cycle` is stored in the `lrutils` module (imported above).\n\nBased on the Cyclic LR strategy described here:<br>\nhttp://course18.fast.ai/lessons/lesson2.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_lrs(max_lr, epochs, batches_per_epoch, cycle_mult=1):\n    lrs = []\n    cycle_iterations = 0\n    epoch_iterations = batches_per_epoch\n    for _ in range(epochs*batches_per_epoch):\n        decay_phase = np.pi*cycle_iterations/epoch_iterations\n        decay = (np.cos(decay_phase) + 1.) / 2.\n        lrs.append(max_lr*decay)\n        cycle_iterations += 1\n        if cycle_iterations == epoch_iterations:\n            cycle_iterations = 0\n            epoch_iterations *= cycle_mult\n    return lrs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"During training, the learning rate will look like this:"},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = compute_lrs(0.0014, 10, 450)\nplt.plot(np.arange(10*450), lrs)\nplt.xlabel('Batch number')\nplt.ylabel('Learning rate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet = keras.Sequential([\n    mobilenet,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(1, activation='sigmoid')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.compile(optimizer=keras.optimizers.RMSprop(learning_rate=tuned_learning_rate),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let the cycle length be one epoch. There are 450 batches in one training epoch (22500 training images / 50 images per batch), so:"},{"metadata":{"trusted":true},"cell_type":"code","source":"cyclic_lr = LR_Cycle(450)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"savebest = keras.callbacks.ModelCheckpoint('cyclic_weights.h5', monitor='val_loss', mode='min',\n                                             save_best_only=True, save_weights_only=True,\n                                             verbose=1, save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = classifier_mobilenet.fit_generator(train_generator,\n                              epochs=10,\n                              validation_data=validation_generator,\n                                callbacks=[cyclic_lr, savebest])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.load_weights('cyclic_weights.h5')\nreport(classifier_mobilenet, history, validation_generator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cycle_mult = 2\n\nSetting cycle_mult=2 and running for 15 epochs, the learning rate looks like this:"},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = compute_lrs(0.0014, 15, 450, 2)\nplt.plot(np.arange(15*450), lrs)\nplt.xlabel('Batch number')\nplt.ylabel('Learning rate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Over 15 epochs, the learning rate goes through 4 complete annealing cycles, of lengths 1, 2, 4, and finally 8."},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet = keras.Sequential([\n    mobilenet,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(1, activation='sigmoid')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.compile(optimizer=keras.optimizers.RMSprop(learning_rate=0.0014),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cyclic_lr = LR_Cycle(450, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"savebest = keras.callbacks.ModelCheckpoint('cycleMult2_weights.h5', monitor='val_loss', mode='min',\n                                             save_best_only=True, save_weights_only=True,\n                                             verbose=1, save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = classifier_mobilenet.fit_generator(train_generator,\n                              epochs=15,\n                              validation_data=validation_generator,\n                                callbacks=[cyclic_lr, savebest])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier_mobilenet.load_weights('cycleMult2_weights.h5')\nreport(classifier_mobilenet, history, validation_generator)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}