{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-20T16:46:17.148307Z","iopub.execute_input":"2023-03-20T16:46:17.148833Z","iopub.status.idle":"2023-03-20T16:46:27.519657Z","shell.execute_reply.started":"2023-03-20T16:46:17.148785Z","shell.execute_reply":"2023-03-20T16:46:27.517899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's unzip the training data\n!unzip -qq /kaggle/input/dogs-vs-cats/train.zip","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:38:17.144610Z","iopub.execute_input":"2023-03-02T15:38:17.145493Z","iopub.status.idle":"2023-03-02T15:38:29.226892Z","shell.execute_reply.started":"2023-03-02T15:38:17.145443Z","shell.execute_reply":"2023-03-02T15:38:29.225466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, shutil, pathlib ","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:38:34.063530Z","iopub.execute_input":"2023-03-02T15:38:34.064027Z","iopub.status.idle":"2023-03-02T15:38:34.069655Z","shell.execute_reply.started":"2023-03-02T15:38:34.063985Z","shell.execute_reply":"2023-03-02T15:38:34.068273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#We'll use these directories to create our training, validation and test sets. \n\noriginal_dir = pathlib.Path(\"/kaggle/working/train\") #where our unzipped data resides\nnew_base_dir = pathlib.Path(\"/kaggle/working/cats_vs_dogs_small\") #where we want to store our smaller dataset\n","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:38:46.360699Z","iopub.execute_input":"2023-03-02T15:38:46.361164Z","iopub.status.idle":"2023-03-02T15:38:46.367803Z","shell.execute_reply.started":"2023-03-02T15:38:46.361122Z","shell.execute_reply":"2023-03-02T15:38:46.366743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function to create our datasets \n\ndef make_subset(subset_name, start_index, end_index):\n    for category in (\"cat\", \"dog\"):\n        dir = new_base_dir/subset_name/category\n        os.makedirs(dir)\n        fnames = [f\"{category}.{i}.jpg\" for i in range(start_index, end_index)]\n        for fname in fnames:\n            shutil.copyfile(src=original_dir / fname,\n                            dst=dir/fname)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:39:00.187997Z","iopub.execute_input":"2023-03-02T15:39:00.188443Z","iopub.status.idle":"2023-03-02T15:39:00.197042Z","shell.execute_reply.started":"2023-03-02T15:39:00.188405Z","shell.execute_reply":"2023-03-02T15:39:00.195235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Now we can use our make_subset function to create our datasets\n#We now have 2000 training, 1000 validation and 2000 testing images.\nmake_subset(\"train\", start_index=0, end_index=1000)\nmake_subset(\"validation\", start_index=1000, end_index=1500)\nmake_subset(\"test\", start_index=1500, end_index=2500)","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:39:05.631829Z","iopub.execute_input":"2023-03-02T15:39:05.632250Z","iopub.status.idle":"2023-03-02T15:39:06.423193Z","shell.execute_reply.started":"2023-03-02T15:39:05.632213Z","shell.execute_reply":"2023-03-02T15:39:06.421807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:39:25.195980Z","iopub.execute_input":"2023-03-02T15:39:25.196425Z","iopub.status.idle":"2023-03-02T15:39:25.203173Z","shell.execute_reply.started":"2023-03-02T15:39:25.196384Z","shell.execute_reply":"2023-03-02T15:39:25.201653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Time to build our CovNet \ninputs = keras.Input(shape = (180,180,3)) #Our model will now expect RGB images of size 180x180 pixels.\nx = layers.Rescaling(1./255)(inputs) #recaling our images from 0-255, to 0-1\nx = layers.Conv2D(filters = 32, kernel_size = 3, activation = \"relu\")(x)\nx = layers.MaxPooling2D(pool_size = 2)(x)\nx = layers.Conv2D(filters = 64, kernel_size = 3, activation = \"relu\")(x)\nx = layers.MaxPooling2D(pool_size = 2)(x)\nx = layers.Conv2D(filters = 128, kernel_size = 3, activation = \"relu\")(x)\nx = layers.MaxPooling2D(pool_size = 2)(x)\nx = layers.Conv2D(filters = 256, kernel_size = 3, activation = \"relu\")(x)\nx = layers.MaxPooling2D(pool_size = 2)(x)\nx = layers.Conv2D(filters=256, kernel_size=3, activation=\"relu\")(x) \nx = layers.Flatten()(x)\n\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = keras.Model(inputs=inputs, outputs=outputs)","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:39:27.980241Z","iopub.execute_input":"2023-03-02T15:39:27.980661Z","iopub.status.idle":"2023-03-02T15:39:28.295722Z","shell.execute_reply.started":"2023-03-02T15:39:27.980621Z","shell.execute_reply":"2023-03-02T15:39:28.294465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The code above defines a convolutional neural network (CNN) using the Keras library. The CNN takes images of size 180x180x3 as inputs, where 3 is the number of color channels (RGB).\n\nThe model consists of a series of layers that transform the input image through a series of convolutions, pooling, and flattening operations, and finally a dense layer that produces a single output value that indicates whether the input image belongs to a specific class or not.\n\nThe Rescaling layer scales the pixel values of the input image by a factor of 1/255, so that the pixel values are between 0 and 1. This is a common preprocessing step that helps the model to learn more effectively. \nScaling every images to the same range [0,1] will make images contributes more evenly to the total loss. Without scaling, the high pixel range images will have a large say to determine how to update weights.\n\nThe next series of layers consist of Conv2D and MaxPooling2D layers, which perform convolution and pooling operations on the input image respectively. Convolutional layers apply a set of filters to the input image, extracting features that are important for classification. MaxPooling layers downsample the feature maps obtained from convolutional layers, reducing the size of the input and making the network more computationally efficient.\n\nAfter several convolutional and pooling layers, the last Conv2D layer outputs a feature map of size (3,3,256). This is then flattened into a 1D array using the Flatten layer, which is then fed into a fully connected (Dense) layer with a single output unit and a sigmoid activation function. The output value of the model is a probability between 0 and 1, indicating the likelihood that the input image belongs to the target class.\n\nOverall, this model architecture is designed for binary classification tasks, where the output of the model indicates whether the input image belongs to a specific class or not.","metadata":{}},{"cell_type":"code","source":"#Let's have a look at the model structure\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:43:41.380263Z","iopub.execute_input":"2023-03-02T15:43:41.381678Z","iopub.status.idle":"2023-03-02T15:43:41.432236Z","shell.execute_reply.started":"2023-03-02T15:43:41.381624Z","shell.execute_reply":"2023-03-02T15:43:41.431010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The filter size of a convolutional neural network (CNN) determines the receptive field of the filters, which is the region of the input image that each filter \"sees\". By increasing the filter size in successive layers, the model can capture increasingly complex and higher-level features of the input image.\n\nIn the given code, the CNN starts with a filter size of 32 in the first Conv2D layer, then doubles the filter size in each successive layer, up to 256 filters in the last layer. This progressive increase in filter size allows the model to learn more complex and abstract features from the input image as it progresses through the layers.\n\nMoreover, the increasing filter size can also help the model to learn more discriminative features for the classification task, as the higher-level filters can capture more complex patterns in the input image that are specific to the target class. However, it is important to note that increasing the number of filters also increases the number of model parameters, which can make the model more computationally expensive to train and prone to overfitting if not properly regularized.\n\n\n\n\n","metadata":{}},{"cell_type":"code","source":"#Let's set up the compiler \nmodel.compile(loss = \"binary_crossentropy\", \n             optimizer = \"rmsprop\", \n             metrics = [\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-02T15:46:40.974352Z","iopub.execute_input":"2023-03-02T15:46:40.974813Z","iopub.status.idle":"2023-03-02T15:46:40.998979Z","shell.execute_reply.started":"2023-03-02T15:46:40.974772Z","shell.execute_reply":"2023-03-02T15:46:40.997730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Currently, our images are jpeg format. We'll have to convert them into floating point tensors. We can then feed these floating-point tensors into the model. Here's what we'll have to do, step by step:\n\n1. Read the picture files.\n2. Decode the JPEG content to RGB grids of pixels.\n3. Convert these into floating-point tensors.\n4. Resize them to a shared size (we’ll use 180 × 180).\n5. Pack them into batches (we’ll use batches of 32 images).\n\nKeras has a inbuilt functionality that'll allow you to easily perform these tasks: we'll use the \"image_dataset_from_directory()\" function. Which allows you to turn image files into batches of preprocessed tensors, ready to be fed into a neural network.\n\nThe function assumes that your directory contains a subdirectory for each class, and that the name of each subdirectory is the name of the class. For example, if you have a directory of images of cats and dogs, you would organize your directory like this:\n\n\n`dataset/\n    cats/\n        cat1.jpg\n        cat2.jpg\n        ...\n    dogs/\n        dog1.jpg\n        dog2.jpg\n        ... `","metadata":{}},{"cell_type":"code","source":"#Let's get started by importing the method\nfrom tensorflow.keras.utils import image_dataset_from_directory","metadata":{"execution":{"iopub.status.busy":"2023-03-02T16:03:37.819692Z","iopub.execute_input":"2023-03-02T16:03:37.821054Z","iopub.status.idle":"2023-03-02T16:03:37.826473Z","shell.execute_reply.started":"2023-03-02T16:03:37.821006Z","shell.execute_reply":"2023-03-02T16:03:37.825545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#We'll use this method to create our respective datasets\ntrain_dataset = image_dataset_from_directory(\n            new_base_dir / \"train\",\n            image_size=(180, 180),\n            batch_size=32)\n\nvalidation_dataset = image_dataset_from_directory(\n            new_base_dir / \"validation\",\n            image_size=(180, 180),\n            batch_size=32)\n\ntest_dataset = image_dataset_from_directory(\n            new_base_dir / \"test\",\n            image_size=(180, 180),\n            batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2023-03-02T16:05:58.758580Z","iopub.execute_input":"2023-03-02T16:05:58.759024Z","iopub.status.idle":"2023-03-02T16:05:59.219839Z","shell.execute_reply.started":"2023-03-02T16:05:58.758986Z","shell.execute_reply":"2023-03-02T16:05:59.218420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`import tensorflow as tf`\n\n`train_dataset = tf.keras.preprocessing.image_dataset_from_directory(\n    'dataset',\n    validation_split=0.2,\n    subset='training',\n    seed=123,\n    image_size=(128, 128),\n    batch_size=32)\n`\n\nIn this example, the image_dataset_from_directory() function will read in all the images in the dataset directory and create a dataset of (image, label) pairs, where the label is inferred from the subdirectory name. The validation_split parameter specifies the fraction of the dataset to use for validation, and the subset parameter specifies whether to use the training or validation subset of the data. The image_size parameter specifies the size of the images to be returned by the dataset, and the batch_size parameter specifies the size of the batches to use during training.\n\n\"TensorFlow makes available the **tf.data** API to create efficient input pipelines for machine learning models. Its core class is **tf.data.Dataset**.\nA Dataset object is an iterator: you can use it in a for loop. It will typically return batches of input data and labels. You can pass a Dataset object directly to the **fit()** method of a Keras model.\nThe Dataset class handles many key features that would otherwise be cumbersome to implement yourself—in particular, asynchronous data prefetching (preprocessing the next batch of data while the previous one is being handled by the model, which keeps execution flowing without interruptions).\"","metadata":{}},{"cell_type":"code","source":"#Let's view the shape of our batches, and labels for a bit more clarity. \n\nfor data_batch, labels_batch in train_dataset:\n    print(f'data batch shape: {data_batch.shape}')\n    print(f'labels batch shape: {labels_batch.shape}')\n    break ","metadata":{"execution":{"iopub.status.busy":"2023-03-02T16:07:21.176158Z","iopub.execute_input":"2023-03-02T16:07:21.176804Z","iopub.status.idle":"2023-03-02T16:07:21.184383Z","shell.execute_reply.started":"2023-03-02T16:07:21.176764Z","shell.execute_reply":"2023-03-02T16:07:21.183227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Time to set our keras callbacks, note how they're passing into a list. \n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\n    filepath = 'covnet_from_scratch.keras',\n    save_best_only = True,\n    monitor = \"val_loss\")\n]\n\n#When passed into the \"fit\" method, these callbacks will enable us to save to file, the most recent version of the model with the lowest \"val_loss\".\n#Will save us from manually having to train the model for the complete range of epochs and selecting the best one(with the lowest val_loss).","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Now we use use the fit method on our dataset object. \n#We'll use the validation_data \"object\" we created above to gauge model fitness\n\nhistory = model.fit(\n        train_dataset, \n        epochs = 30,\n        validation = validation_dataset,\n        callbacks = callbacks)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's plot our loss and accuracy of our model per epoch\n\naccuracy = history.history[\"accuracy\"]\nval_accuracy = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\nepochs = range(1, len(accuracy) + 1)\nplt.plot(epochs, accuracy, \"bo\", label=\"Training accuracy\") plt.plot(epochs, val_accuracy, \"b\", label=\"Validation accuracy\") plt.title(\"Training and validation accuracy\")\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, \"bo\", label=\"Training loss\") plt.plot(epochs, val_loss, \"b\", label=\"Validation loss\") plt.title(\"Training and validation loss\")\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"These plots are characteristic of overfitting. The training accuracy increases linearly over time, until it reaches nearly 100%, whereas the validation accuracy peaks at 75%. The validation loss reaches its minimum after only ten epochs and then stalls, whereas the training loss keeps decreasing linearly as training proceeds.\"\nLet’s check the test accuracy. We’ll reload the model from its saved file to evaluate it as it was before it started overfitting.","metadata":{}},{"cell_type":"code","source":"test_model = keras.models.load_model(\"convnet_from_scratch.keras\") \ntest_loss, test_acc = test_model.evaluate(test_dataset) \nprint(f\"Test accuracy: {test_acc:.3f}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data augmentation\n\nIn order get the most out of our input data, which in the real world may be limited. We can use data augmentation. This is a techniques that we can use on our input data prior to feeding into our model. Data augmentation in effect remixes the input data. Keras provides a lot of built in options: rotation, flip, zoom etc. You can also select by how much you want to augment your images. \n\n\"Given infinite data, your model would be exposed to every possible aspect of the data distribution at hand: you would never overfit. Data augmentation takes the approach of generating more training data from existing training samples by augmenting the samples via a number of random transformations that yield believable-looking images. The goal is that, at training time, your model will never see the exact same picture twice. This helps expose the model to more aspects of the data so it can generalize better.\nIn Keras, this can be done by adding a number of data augmentation layers at the start of your model.\" Deep learning with Python (Francois Challot).","metadata":{}},{"cell_type":"code","source":"#Let's first create our data augmentation layer\ndata_augmentation = keras.Sequential(\n            [\n                layers.RandomFlip(\"horizontal\"),\n                layers.RandomRotation(0.1),\n                layers.RandomZoom(0.2),\n            ] )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- RandomFlip(\"horizontal\")—Applies horizontal flipping to a random 50% of the images that go through it\n- RandomRotation(0.1)—Rotates the input images by a random value in the range [–10%, +10%] (these are fractions of a full circle—in degrees, the range would be [–36 degrees, +36 degrees])\n- RandomZoom(0.2)—Zooms in or out of the image by a random factor in the range [-20%, +20%]\n","metadata":{}},{"cell_type":"code","source":"#We can view a grid of the same image from our training set, \"augmented\" as follows:\n#(We can use take(N) to only sample N batches from the dataset. This is equivalent to inserting a break in the loop after the Nth batch)\n#The loop below will display the first image, for 9 iterations. \nplt.figure(figsize=(10, 10))\nfor images, _ in train_dataset.take(1):\n    for i in range(9):\n        augmented_images = data_augmentation(images)\n        ax = plt.subplot(3, 3, i + 1) \n        plt.imshow(augmented_images[0].numpy().astype(\"uint8\")) \n        plt.axis(\"off\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- \"If we train a new model using this data-augmentation configuration, the model will never see the same input twice. But the inputs it sees are still heavily intercorrelated because they come from a small number of original images—we can’t produce new information; we can only remix existing information. As such, this may not be enough to completely get rid of overfitting. To further fight overfitting, we’ll also add a Dropout layer to our model right before the densely connected classifier.\n- One last thing you should know about random image augmentation layers: just like Dropout, they’re inactive during inference (when we call predict() or evaluate()). During evaluation, our model will behave just the same as when it did not include data augmentation and dropout. \" (DLWP, Callot)\n- Data augmentation also allows you to train your CovNet for longer epochs, as you'd expect it overfit much \"later\" during the training procedure. ","metadata":{}},{"cell_type":"code","source":"#Let's now include Data Augmentation in our original covnet \ninputs = keras.Input(shape=(180, 180, 3))\nx = data_augmentation(inputs)\nx = layers.Rescaling(1./255)(x)\nx = layers.Conv2D(filters=32, kernel_size=3, activation=\"relu\")(x)\nx = layers.MaxPooling2D(pool_size=2)(x)\nx = layers.Conv2D(filters=64, kernel_size=3, activation=\"relu\")(x)\nx = layers.MaxPooling2D(pool_size=2)(x)\nx = layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\")(x)\nx = layers.MaxPooling2D(pool_size=2)(x)\nx = layers.Conv2D(filters=256, kernel_size=3, activation=\"relu\")(x)\nx = layers.MaxPooling2D(pool_size=2)(x)\nx = layers.Conv2D(filters=256, kernel_size=3, activation=\"relu\")(x)\nx = layers.Flatten()(x)\nx = layers.Dropout(0.5)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x) \nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(loss=\"binary_crossentropy\",\n              optimizer=\"rmsprop\",\n              metrics=[\"accuracy\"])\n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\n        filepath=\"convnet_from_scratch_with_augmentation.keras\",\n        save_best_only=True,\n        monitor=\"val_loss\")\n]\nhistory = model.fit(\n    train_dataset,\n    epochs=100,\n    validation_data=validation_dataset,\n    callbacks=callbacks)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's look at the loss and accuracy again\n\nimport matplotlib.pyplot as plt\naccuracy = history.history[\"accuracy\"]\nval_accuracy = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\nepochs = range(1, len(accuracy) + 1)\nplt.plot(epochs, accuracy, \"bo\", label=\"Training accuracy\") plt.plot(epochs, val_accuracy, \"b\", label=\"Validation accuracy\") plt.title(\"Training and validation accuracy\")\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, \"bo\", label=\"Training loss\") plt.plot(epochs, val_loss, \"b\", label=\"Validation loss\") plt.title(\"Training and validation loss\")\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks to data augmentation and drop- out, we start overfitting much later, around epochs 60–70 (compared to epoch 10 for the original model). \nThe validation accuracy ends up consistently in the 80–85% range a big improvement over our first try.","metadata":{}},{"cell_type":"code","source":"#Let's evalutate the model with integrated data-augmention on the test set \ntest_model = keras.models.load_mode(\"convnet_from_scratch_with_augmentation.keras\")\n\ntest_loss, test_acc = test_model.evaluate(test_dataset)\n\nprint(f\"test accuracy: {test_acc:.3f}\") \n\n#As we have limited data, it'll be difficult to get a higher performance. \n#This is where pre-trained models come in.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-Trained Models for better performance\n\n- Pretrained models, are usually trained on very large datasets, in the hope the parameters they learn are can be useful in other, in this case Image classification tasks. Being trained on a large model should help it learn spatial hierarchies that apply to new unseen images.\n- We'll use the \"VGG16\" pretrained model, it was trained on the large ImageNet dataset, which contains animals and objects comprising 1000 different classes. In total this dataset contains 1.4m images. \n- As it contains animals, it should be able to identify the cats and dogs in our dataset. \n- This is a huge benefit of deep learning, the ability to repurpose pre-trained models for more nuanced tasks. \n\nPretrained models can be used in 2 ways: Feature extraction and Fine-tuning. \n- In Feature extraction, the pretrained model is used to extract \"interesting\" features, captured via the parameters it learned when it was trained(on a large dataset). These extracted features can then be used as input into a new model, that is trained from scratch.\n- Features are extracted via the pre-trained model's convolutional base.\n- As you saw previously, convnets used for image classification comprise two parts: they start with a series of pooling and convolution layers, and they end with a densely connected classifier. The first part is called the convolutional base of the model. In the case of convnets, feature extraction consists of taking the convolutional base of a pre- viously trained network, running the new data through it, and training a new classifier on top of the output (see figure 8.12).\n-\"Why only reuse the convolutional base? Could we reuse the densely connected classifier as well? In general, doing so should be avoided. The reason is that the repre- sentations learned by the convolutional base are likely to be more generic and, there- fore, more reusable: the feature maps of a convnet are presence maps of generic concepts over a picture, which are likely to be useful regardless of the computer vision problem at hand. But the representations learned by the classifier will necessarily be specific to the set of classes on which the model was trained—they will only contain information about the presence probability of this or that class in the entire picture. Additionally, representations found in densely connected layers no longer contain any information about where objects are located in the input image; these layers get rid of the notion of space, whereas the object location is still described by convolutional fea- ture maps. For problems where object location matters, densely connected features are largely useless.\nNote that the level of generality (and therefore reusability) of the representations extracted by specific convolution layers depends on the depth of the layer in the model. Layers that come earlier in the model extract local, highly generic feature maps (such as visual edges, colors, and textures), whereas layers that are higher up extract more-abstract concepts (such as “cat ear” or “dog eye”). So if your new dataset differs a lot from the dataset on which the original model was trained, you may be bet- ter off using only the first few layers of the model to do feature extraction, rather than using the entire convolutional base.\"","metadata":{}},{"cell_type":"markdown","source":"## There are 2 methods available for using a pretrained convolutional base for feature-extraction. \n\n- The first consists of freezing the weights of the convolutional base, passing our data through the base, and using the extracted features as input to a new dense layer. This method is fast, as the images are only run through the base once, and we only need to train the final dense layers we appended. However, this method does not allow for data-augmentation. \n- The second method consists of extending the convolutional base, again freezing it's weights, but adding data-augmentation layers prior to inputting into the model. This is more computational expensive. As it requires end to end training. ","metadata":{}},{"cell_type":"code","source":"#Setting up our convolutional base\nconv_base = keras.applications.vgg16.VGG16(\n    weights=\"imagenet\",\n    include_top=False,\n    input_shape=(180, 180, 3)) #shape of the input tensors we'll input into the pre-trained model.\n\n#\"weights\" specifies the weight checkpoint from which to instantiate the model\n#\"include_top\" allows you to include/or not the last densely connected layer of the pre-trained base.\n#VGG16's top dense layer was trained for classifying 1000 classes, we only want to classify 2, so we'll add our own.\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Summary of the architecture of our pre-trained base\n\nconv_base.summary()\n\n#The final feature map has shape (5,5,512)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In general, we can proceed in one of 2 ways once you've set up your pretrained base:\nThe first way: \n\n1. \"Run the convolutional base over our dataset\n2. record its output to a NumPy array on disk,\n3. and then use this data as input to a standalone, densely connected classifier. \n- This solution is fast and cheap to run, because it only requires running the convolutional base once for every input image, and the convolutional base is by far the most expensive part of the pipeline. But for the same reason, this technique won’t allow us to use data augmentation.\"\n\nThe second way: \n1. \"Extend the model we have (conv_base) by adding Dense layers on top, and run the whole thing from end to end on the input data. \n- This will allow us to use **data augmentation**, because every input image goes through the convolutional base every time it’s seen by the model. But for the same reason, this technique is far more expensive than the first.\" (Challot, DLWP)\n\nWe'll start with first method: Feature extraction without data-augmentation.","metadata":{}},{"cell_type":"code","source":"#Feature extraction without data-augmentation \n#Let's first define a function to extract features via VGG16 from our training, validation and testing datasets.\n\ndef get_features_and_labels(dataset): \n    all_features = []\n    all_labels = []\n    for images, labels in dataset:\n        preprocessed_images = keras.applications.vgg16.preprocess_input(images) \n        features = conv_base.predict(preprocessed_images) \n        all_features.append(features)\n        all_labels.append(labels)\nreturn np.concatenate(all_features), np.concatenate(all_labels)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Using our function on our datasets\ntrain_features, train_labels = get_features_and_labels(train_dataset) \nval_features, val_labels = get_features_and_labels(validation_dataset) \ntest_features, test_labels = get_features_and_labels(test_dataset)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"\nImportantly, predict() only expects images, not labels, but our current dataset yields batches that contain both images and their labels. Moreover, the VGG16 model expects inputs that are preprocessed with the function keras.applications.vgg16.preprocess_input, which scales pixel values to an appropriate range. \"","metadata":{}},{"cell_type":"code","source":"#We know that the features extracted from the convolutional base should be of shape (5,5,512)\ntrain_features.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Now we can create our densely connected classifier. \n#We'll have to use a Flatten() layer before we pass our extracted features to the dense layer\ninputs = keras.Input(shape=(5, 5, 512))\nx = layers.Flatten()(inputs)\nx = layers.Dense(256)(x)\nx = layers.Dropout(0.5)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = keras.Model(inputs, outputs)\n\nmodel.compile(loss=\"binary_crossentropy\",\n                      optimizer=\"rmsprop\",\n                      metrics=[\"accuracy\"])\n\ncallbacks = [\n            keras.callbacks.ModelCheckpoint(\n                filepath=\"feature_extraction.keras\",\n                save_best_only=True,\n                monitor=\"val_loss\")\n       ]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting our model\nhistory = model.fit(\n            train_features, train_labels,\n            epochs=20,\n            validation_data=(val_features, val_labels),\n            callbacks=callbacks)\n\n#This feature extraction method allows the model to trained quickly, as there's only 2 Dense Layers.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's see how our model performed per epoch, in terms of loss and accuracy\n\nacc = history.history[\"accuracy\"]\nval_acc = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\nepochs = range(1, len(acc) + 1)\nplt.plot(epochs, acc, \"bo\", label=\"Training accuracy\") \nplt.plot(epochs, val_acc, \"b\", label=\"Validation accuracy\") \nplt.title(\"Training and validation accuracy\")\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, \"bo\", label=\"Training loss\")\nplt.plot(epochs, val_loss, \"b\", label=\"Validation loss\")\nplt.title(\"Training and validation loss\")\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The Feature extraction method reached a validation accuracy of 97%. Which is quite good, but the plots also indicate that the model began overfitting within 10 epochs. One of the reasons for this is that this method doesn't allow for the use of data augmentation. \n\nWe'll now implement the second more expensive approach that allows for data augmentation. This method basically extends the convolutional base to a pipeline that allows for data augmentation, and training it end to end on the inputs. \n- We'll need to make sure that we freeze the weights of the convultional base, we want to keep the representations it learned. If we don't do this, you're basically overwriting the information that was gained when the model was pre-trained. (\"Because the Dense layers on top are randomly initialized, very large weight updates would be propagated through the network, effectively destroying the representations previously learned.\").\n- We can freeze a layer in Keras by setting it's `trainable` attribute to `False`.","metadata":{}},{"cell_type":"code","source":"conv_base  = keras.applications.vgg16.VGG16(\n    weights=\"imagenet\",\n    include_top=False)\n\nconv_base.trainable = False\n#Setting trainable to False empties the list of trainable weights of the layer or model.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can check that the number of trainable weights is zero via the following code.\n\nconv_base.trainable = True\n- print(\"This is the number of trainable weights \"\"before freezing the conv base:\", len(conv_base.trainable_weights)) \n- output: This is the number of trainable weights before freezing the conv base: 26 \n\n- conv_base.trainable = False\n- print(\"This is the number of trainable weights \"\"after freezing the conv base:\", len(conv_base.trainable_weights)) \n- output: This is the number of trainable weights after freezing the conv base: 0","metadata":{}},{"cell_type":"markdown","source":"Now we can create a new model that chains together:\n\n1. A data augmentation stage\n2. Our frozen convolutional base\n3. A dense classifier","metadata":{}},{"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        layers.RandomFlip(\"horizontal\"),\n        layers.RandomRotation(0.1),\n        layers.RandomZoom(0.2),\n] )\n\ninputs = keras.Input(shape=(180, 180, 3))\nx = data_augmentation(inputs) #adding data-augmentation\nx = keras.applications.vgg16.preprocess_input(x) #applying input value scaling\nx = conv_base(x)\nx = layers.Flatten()(x)\nx = layers.Dense(256)(x)\nx = layers.Dropout(0.5)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = keras.Model(inputs, outputs)\n\n\nmodel.compile(loss=\"binary_crossentropy\",\n              optimizer=\"rmsprop\",\n              metrics=[\"accuracy\"])\n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\n        filepath=\"feature_extraction_with_data_augmentation.keras\",\n        save_best_only=True,\n        monitor=\"val_loss\")\n    \n    #Fitting the model\n    \n    history = model.fit(\n            train_dataset,\n            epochs=50,\n            validation_data=validation_dataset,\n            callbacks=callbacks)\n]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"With this setup, only the weights from the two Dense layers that we added will be trained. That’s a total of four weight tensors: two per layer (the main weight matrix and the bias vector). Note that in order for these changes to take effect, you must first compile the model. If you ever modify weight trainability after compilation, you should then recompile the model, or these changes will be ignored.\"\n\nWith data augmentation now added, we can train the model for larger epochs, as we don't have to worry about it overfitting so soon after training.\n\nTo train this model, you'll need to use a GPU. As it's quite expensive to run.\n","metadata":{}},{"cell_type":"code","source":"#Lets plot the per epoch loss and accuracy for the training and validation set again\nacc = history.history[\"accuracy\"]\nval_acc = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\nepochs = range(1, len(acc) + 1)\nplt.plot(epochs, acc, \"bo\", label=\"Training accuracy\") \nplt.plot(epochs, val_acc, \"b\", label=\"Validation accuracy\") \nplt.title(\"Training and validation accuracy\")\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, \"bo\", label=\"Training loss\")\nplt.plot(epochs, val_loss, \"b\", label=\"Validation loss\")\nplt.title(\"Training and validation loss\")\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluting on the test set\ntest_model = keras.models.load_model(\n            \"feature_extraction_with_data_augmentation.keras\")\ntest_loss, test_acc = test_model.evaluate(test_dataset) \nprint(f\"Test accuracy: {test_acc:.3f}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now we'll show how to use the convolutional base for fine-tuning.\n\n- Unlike the feature-extraction method, this method unfreezes the weights of the topmost convolutional layers of the conv_base. \n- \"...it’s only possible to fine-tune the top layers of the convolutional base once the classifier on top has already been trained. If the classifier isn’t already trained, the error signal propagating through the network during training will be too large, and the representations previously learned by the layers being fine-tuned will be destroyed.\"\nHere are the steps: \n1. Add our custom network on top of an already-trained base network.\n2. Freeze the base network.\n3. Train the part we added.\n4. Unfreeze some layers in the base network.(Note that you should not unfreeze “batch normalization” layers, which are not relevant here since there are no such layers in VGG16. Batch normalization and its impact on fine- tuning is explained in the next chapter.)\n5. Jointly train both these layers and the part we added.\n\nWe generally only want to fine-tune the topmost layers of the conv_base. The earlier layers of the pre-trained models learn generic-reusable features, so we want to keep these. Layers higher up encode representations of the more finer details of images, so these are the layers that we'll want to fine-tune to our task.  \nThere's also the number of parameters you're training, depending on the pretrained model, there'll generally be millions of parameters. So training deeper layers doesn't make sense, especially on a small dataset. The risk of overfitting will be increased.\n\nGenerally, we should aim to fine-tune the top 3 or 4 layers of the convolutional base. And we'll keep the learning rate very small, as we just want to slightly tweak the pre-trained model for our dataset, we don't want to harm useful represations it may have learning when it was pre-trained.","metadata":{}},{"cell_type":"code","source":"#Let's remind ourselves of the convolutional base\n#Note, the number of parameters!\nconv_base.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**\"We’ll fine-tune the last three convolutional layers, which means all layers up to block4_pool should be frozen, and the layers block5_conv1, block5_conv2, and block5_conv3 should be trainable.\"**","metadata":{}},{"cell_type":"code","source":"#Freezing the layers except the ones we're gna fine-tune\nconv_base.trainable = True\nfor layer in conv_base.layers[:-4]:\n    layer.trainable = False","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Specifying our CovNet for fine-tuning.\n\nmodel.compile(loss=\"binary_crossentropy\",\n              optimizer=keras.optimizers.RMSprop(learning_rate=1e-5),\n              metrics=[\"accuracy\"])\n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\n        filepath=\"fine_tuning.keras\",\n        save_best_only=True,\n        monitor=\"val_loss\")\n]\n\nhistory = model.fit(\n    train_dataset,\n    epochs=30,\n    validation_data=validation_dataset,\n    callbacks=callbacks)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluating our fine-tuned model on test-data.\nmodel = keras.models.load_model(\"fine_tuning.keras\") \ntest_loss, test_acc = model.evaluate(test_dataset) \nprint(f\"Test accuracy: {test_acc:.3f}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}