{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Image Classification : Dog or Cat\nClassifying image is of dog or cat using Computer vision. \n\nThis notebook covers topics like:\n* Classification through Convolutional Network\n* Data Augmentation for Computer vision classification problem\n* Transfer learning - Using pretrained models for feature extraction and fine tuning\n* Visualizing pattterns learned by Convnet model\n\n\nThe Image Classification steps include:\n* Downloading the data.\n* Preprocessing the data and Visualize.\n* Building the model\n* Evaluating and visualize\n* Using Data augmentation\n* Using Transfer learning for feature extraction and fine tuning\n\nThis example is from Deep learning with python book.\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"### Downloading the data\n* The dataset dogs vs cats is from kaggle competition \n* It contains 25,000 images of dogs anf cats (12,500 for each class)\n* In this we will use 10% of data viz. 1000 images for training, 500 images for validation and 500 images for testing.","metadata":{}},{"cell_type":"code","source":"import zipfile\n\nzip_ref = zipfile.ZipFile(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip\", \"r\")\nzip_ref.extractall()\nzip_ref.close()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:25:46.983632Z","iopub.execute_input":"2023-09-23T07:25:46.983999Z","iopub.status.idle":"2023-09-23T07:25:59.181057Z","shell.execute_reply.started":"2023-09-23T07:25:46.983969Z","shell.execute_reply":"2023-09-23T07:25:59.179890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the GPU\n!nvidia-smi -L","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:25:59.183949Z","iopub.execute_input":"2023-09-23T07:25:59.184937Z","iopub.status.idle":"2023-09-23T07:26:00.228897Z","shell.execute_reply.started":"2023-09-23T07:25:59.184899Z","shell.execute_reply":"2023-09-23T07:26:00.227626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Copying images to training, validation and test directories","metadata":{}},{"cell_type":"code","source":"import os, shutil\n\noriginal_dataset_dir = \"/kaggle/working/train\"\n\n# 10% data directory\nbase_dir = \"/kaggle/working/cats_and_dogs_small\"\nos.mkdir(base_dir)\n\n# Make train, validation, test, dir\ntrain_dir = os.path.join(base_dir, 'train')\nos.mkdir(train_dir)\n\nvalidation_dir = os.path.join(base_dir, 'validation')\nos.mkdir(validation_dir)\n\ntest_dir = os.path.join(base_dir, 'test')\nos.mkdir(test_dir)\n\n# Directory with training cats and dogs picture\ntrain_cats_dir = os.path.join(train_dir, 'cats')\nos.mkdir(train_cats_dir)\n\ntrain_dogs_dir = os.path.join(train_dir, 'dogs')\nos.mkdir(train_dogs_dir)\n\n# Directory with validation cats and dogs pictures\nvalidation_cats_dir = os.path.join(validation_dir, 'cats')\nos.mkdir(validation_cats_dir)\n\nvalidation_dogs_dir = os.path.join(validation_dir, 'dogs')\nos.mkdir(validation_dogs_dir)\n\n# Directory for testing cats and dogs pictures \ntest_cats_dir = os.path.join(test_dir, 'cats')\nos.mkdir(test_cats_dir)\n\ntest_dogs_dir = os.path.join(test_dir, 'dogs')\nos.mkdir(test_dogs_dir)\n\n# Copying the first 1000 cat to train_cats_dir\nfnames = ['cat.{}.jpg'.format(i) for i in range(1000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(train_cats_dir, fname)\n    shutil.copyfile(src, dst)\n\n# Copying the next 500 cat images to validation_cats_dir\nfnames = ['cat.{}.jpg'.format(i) for i in range(1000, 1500)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(validation_cats_dir, fname)\n    shutil.copyfile(src, dst)\n\n\n# Copying the next 500 cat images to test_cats_dir\nfnames = ['cat.{}.jpg'.format(i) for i in range(1500,2000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(test_cats_dir, fname)\n    shutil.copyfile(src, dst)\n\n# Copying the first 1000 dogs images to train_dogs_dir\nfnames = ['dog.{}.jpg'.format(i) for i in range(1000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(train_dogs_dir, fname)\n    shutil.copyfile(src, dst)\n\n# Copying the next 500 dogs images to validation_dogs_dir\nfnames = ['dog.{}.jpg'.format(i) for i in range(1000,1500)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(validation_dogs_dir, fname)\n    shutil.copyfile(src, dst)\n\n# Copying the next 500 dog images to test_dogs_dir \nfnames = ['dog.{}.jpg'.format(i) for i in range(1500, 2000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(test_dogs_dir, fname)\n    shutil.copyfile(src, dst)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:00.231174Z","iopub.execute_input":"2023-09-23T07:26:00.231639Z","iopub.status.idle":"2023-09-23T07:26:00.720068Z","shell.execute_reply.started":"2023-09-23T07:26:00.231573Z","shell.execute_reply":"2023-09-23T07:26:00.718937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing the data and Visualize data\nLet's count how many pictures are in each training split(train/validation/test) ","metadata":{}},{"cell_type":"code","source":"print(\"Total training cat images: \", len(os.listdir(train_cats_dir)))\nprint(\"Total training dog images: \", len(os.listdir(train_cats_dir)))\nprint(\"Total validation cat images: \", len(os.listdir(validation_cats_dir)))\nprint(\"Total validation dog images: \", len(os.listdir(validation_dogs_dir)))\nprint(\"Total test cat images: \", len(os.listdir(test_cats_dir)))\nprint(\"Total test dog images: \", len(os.listdir(test_cats_dir)))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:00.722879Z","iopub.execute_input":"2023-09-23T07:26:00.723191Z","iopub.status.idle":"2023-09-23T07:26:00.734949Z","shell.execute_reply.started":"2023-09-23T07:26:00.723163Z","shell.execute_reply":"2023-09-23T07:26:00.733713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data preprocessing\nSteps of getting images into the network is as follows:\n 1. Read the picture files\n 2. Decode the picture content to RGB grids of pixels\n 3. Convert these into floating-point tesnors\n 4. Rescale the pixel values(between 0 and 255) to the (0 and 1) interval(neural networks prefer to deal with small values)\n ","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\n# Rescale all the image\ntrain_datagen = ImageDataGenerator(rescale=1/255.)\nvalid_datagen = ImageDataGenerator(rescale=1/255.)\ntest_datagen = ImageDataGenerator(rescale=1/255.) \n\n# Instatiate ImageDataGenerator which generates image files into batches of preprocessed tensors\ntrain_generator = train_datagen.flow_from_directory(train_dir,\n                                                   target_size=(150,150),#resizes all images\n                                                   batch_size= 20,\n                                                   class_mode=\"binary\")\n\nvalidation_generator = valid_datagen.flow_from_directory(validation_dir,\n                                                       target_size=(150,150),\n                                                       batch_size=20,\n                                                       class_mode=\"binary\")\n\ntest_generator = test_datagen.flow_from_directory(test_dir,\n                                              target_size=(150,150),\n                                              batch_size=20,\n                                              class_mode=\"binary\",\n                                              shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:39.589152Z","iopub.execute_input":"2023-09-23T07:26:39.589828Z","iopub.status.idle":"2023-09-23T07:26:48.962090Z","shell.execute_reply.started":"2023-09-23T07:26:39.589795Z","shell.execute_reply":"2023-09-23T07:26:48.960994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of batches in train, validation and test_data\nprint(\"No.of batches in train data:\" ,len(train_generator))\nprint(\"No.of batches in validation data: \", len(validation_generator))\nprint(\"No. of batches in test data: \", len(test_generator))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:48.964877Z","iopub.execute_input":"2023-09-23T07:26:48.965733Z","iopub.status.idle":"2023-09-23T07:26:48.973411Z","shell.execute_reply.started":"2023-09-23T07:26:48.965692Z","shell.execute_reply":"2023-09-23T07:26:48.972268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Displaying random images\nCreating a function to display random images","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\ndef preprocess_and_plot_images(directory_path):\n    # Get a list of all image files in the directory\n    image_files = [file for file in os.listdir(directory_path) if file.lower().endswith(('.png', '.jpg', '.jpeg'))]\n     # Select 4 random images from the list\n    random_images = random.sample(image_files, 4)\n    \n    # Create a 2x2 subplot for plotting the images\n    fig, axs = plt.subplots(2, 2, figsize=(10, 10))\n    \n    for i, image_file in enumerate(random_images):\n        # Load the image\n        image_path = os.path.join(directory_path, image_file)\n        image = Image.open(image_path)\n        \n        # Get the image size\n        image_size = image.size\n        \n        # Plot the image on the corresponding subplot\n        ax = axs[i // 2, i % 2]\n        ax.imshow(image)\n        ax.axis('off')\n        ax.set_title(f'Size: {image_size[0]}x{image_size[1]}')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:48.975115Z","iopub.execute_input":"2023-09-23T07:26:48.976209Z","iopub.status.idle":"2023-09-23T07:26:48.987494Z","shell.execute_reply.started":"2023-09-23T07:26:48.976140Z","shell.execute_reply":"2023-09-23T07:26:48.986234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Random cat images\npreprocess_and_plot_images(directory_path=\"/kaggle/working/cats_and_dogs_small/train/cats\")","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:48.989491Z","iopub.execute_input":"2023-09-23T07:26:48.990375Z","iopub.status.idle":"2023-09-23T07:26:50.310902Z","shell.execute_reply.started":"2023-09-23T07:26:48.990336Z","shell.execute_reply":"2023-09-23T07:26:50.309665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the model\nInstantiating a small convnet for dogs vs. cats classification","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom keras import layers\nfrom keras import models\n\nmodel = tf.keras.Sequential([\n    layers.Conv2D(32,(3,3), activation=\"relu\", input_shape=(150,150,3)),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(64, (3,3), activation=\"relu\"),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(128, (3,3), activation=\"relu\"),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(128, (3,3), activation=\"relu\"),\n    layers.MaxPooling2D((2,2)),\n    layers.Flatten(),\n    layers.Dense(512, activation=\"relu\"),\n    layers.Dense(1, activation=\"sigmoid\")\n])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:50.315431Z","iopub.execute_input":"2023-09-23T07:26:50.316935Z","iopub.status.idle":"2023-09-23T07:26:55.800886Z","shell.execute_reply.started":"2023-09-23T07:26:50.316895Z","shell.execute_reply":"2023-09-23T07:26:55.799342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:55.802380Z","iopub.execute_input":"2023-09-23T07:26:55.802799Z","iopub.status.idle":"2023-09-23T07:26:55.837594Z","shell.execute_reply.started":"2023-09-23T07:26:55.802760Z","shell.execute_reply":"2023-09-23T07:26:55.836676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(loss=\"binary_crossentropy\",\n             optimizer=tf.keras.optimizers.Adam(),\n             metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:55.838823Z","iopub.execute_input":"2023-09-23T07:26:55.839431Z","iopub.status.idle":"2023-09-23T07:26:55.871069Z","shell.execute_reply.started":"2023-09-23T07:26:55.839390Z","shell.execute_reply":"2023-09-23T07:26:55.869980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting the model using the batch generator\nhistory = model.fit(\n            train_generator,\n            steps_per_epoch= len(train_generator),\n            epochs= 30,\n            validation_data= validation_generator,\n            validation_steps= len(validation_generator),\n            verbose= 1)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:26:55.872463Z","iopub.execute_input":"2023-09-23T07:26:55.873426Z","iopub.status.idle":"2023-09-23T07:31:10.120551Z","shell.execute_reply.started":"2023-09-23T07:26:55.873390Z","shell.execute_reply":"2023-09-23T07:31:10.119430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving the model\nIt's good practice to always save model after training","metadata":{}},{"cell_type":"code","source":"model.save(\"model/cats_and_dogs_small_1.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:10.122795Z","iopub.execute_input":"2023-09-23T07:31:10.123234Z","iopub.status.idle":"2023-09-23T07:31:10.254851Z","shell.execute_reply.started":"2023-09-23T07:31:10.123193Z","shell.execute_reply":"2023-09-23T07:31:10.253648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate and visualize the model","metadata":{}},{"cell_type":"code","source":"model.evaluate(test_generator)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:10.257548Z","iopub.execute_input":"2023-09-23T07:31:10.258923Z","iopub.status.idle":"2023-09-23T07:31:12.905735Z","shell.execute_reply.started":"2023-09-23T07:31:10.258877Z","shell.execute_reply":"2023-09-23T07:31:12.904673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is clearly seen that the model is only guessing.Hmm... We need to try some other techniques.","metadata":{}},{"cell_type":"markdown","source":"### Displaying curves of loss and accuracy during training","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create a function to plot loss and accuracy curves\n\ndef plot_accuracy_and_loss(history):\n    plt.figure(figsize=(10,7))\n    acc = history.history[\"accuracy\"]\n    val_acc = history.history[\"val_accuracy\"]\n    loss = history.history[\"loss\"]\n    val_loss = history.history[\"val_loss\"]\n\n    epochs = range(1, len(loss) + 1)\n\n    # Plot accuracy curve\n    plt.plot(epochs, acc, 'b', label=\"Training Accuracy\")\n    plt.plot(epochs, val_acc, 'r', label=\"Validation Accuracy\")\n    plt.title(\"Training and Validation accuracy\")\n    plt.legend()\n\n    # Plotting the loss curve\n    plt.figure(figsize=(10,7))\n    plt.plot(epochs, loss, 'b', label=\"Training loss\")\n    plt.plot(epochs, val_loss, 'r', label=\"Validation loss\")\n    plt.title(\"Training and Validation loss\")\n    plt.legend()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:12.907333Z","iopub.execute_input":"2023-09-23T07:31:12.908034Z","iopub.status.idle":"2023-09-23T07:31:12.919872Z","shell.execute_reply.started":"2023-09-23T07:31:12.907994Z","shell.execute_reply":"2023-09-23T07:31:12.916476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracy_and_loss(history=history);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:12.923344Z","iopub.execute_input":"2023-09-23T07:31:12.923692Z","iopub.status.idle":"2023-09-23T07:31:13.623191Z","shell.execute_reply.started":"2023-09-23T07:31:12.923665Z","shell.execute_reply":"2023-09-23T07:31:13.622197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems the model is overfitting after only 5 epochs.. We have only few training samples(2000). So, let's try data augmentation ","metadata":{}},{"cell_type":"markdown","source":"# Using data augmentation \nData augmentation takes the approach of generating more training data from existing training samples, by *augmenting* the samples via a number of random transformations that yield believable-looking images.","metadata":{}},{"cell_type":"markdown","source":"### Setting up data augmentation configuration via ImageDataGenerator","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1/255.,\n                             rotation_range=40,\n                             width_shift_range=0.2,\n                             height_shift_range=0.2,\n                             shear_range=0.2,\n                             zoom_range=0.2,\n                             horizontal_flip=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:13.624711Z","iopub.execute_input":"2023-09-23T07:31:13.625208Z","iopub.status.idle":"2023-09-23T07:31:13.631395Z","shell.execute_reply.started":"2023-09-23T07:31:13.625170Z","shell.execute_reply":"2023-09-23T07:31:13.630221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Displaying some randomly augmented training images","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing import image\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.utils import load_img\nimport tensorflow as tf\nfrom keras import layers\n\ncats_dir = \"/kaggle/working/cats_and_dogs_small/train/cats\"\n\nfnames = [os.path.join(cats_dir, fname) for fname in os.listdir(cats_dir)]\n\nimg_path = fnames[12] # Chooses one images to augment\nimg = load_img(img_path, target_size=(150,150)) # Reads the image and resizes it\n\nx = tf.keras.utils.img_to_array(img) #Converts to Numpy array with shape(150,150,3)\nx = x.reshape((1,) + x.shape) # Reshape it to (1,150,150,3)\n\ni = 0\n#Generates batches of randomly transformed images. Loops indefinitely, so you need to break the loops at some point!\nfor batch in datagen.flow(x, batch_size=1):\n    plt.figure(i)\n    plt.axis(False)\n    img_plot = plt.imshow(tf.keras.utils.array_to_img(batch[0]))\n    i += 1\n    if i % 4 == 0:\n        break \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:13.633204Z","iopub.execute_input":"2023-09-23T07:31:13.633986Z","iopub.status.idle":"2023-09-23T07:31:14.536530Z","shell.execute_reply.started":"2023-09-23T07:31:13.633950Z","shell.execute_reply":"2023-09-23T07:31:14.535549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining a new convnet thst includes that includes dropout","metadata":{"execution":{"iopub.status.busy":"2023-07-15T14:24:30.052807Z","iopub.execute_input":"2023-07-15T14:24:30.053197Z","iopub.status.idle":"2023-07-15T14:24:30.061441Z","shell.execute_reply.started":"2023-07-15T14:24:30.053167Z","shell.execute_reply":"2023-07-15T14:24:30.060341Z"}}},{"cell_type":"code","source":"model_1 = models.Sequential([\n    layers.Conv2D(32,(3,3), activation=\"relu\", input_shape=(150,150,3)),\n    layers.MaxPooling2D((2,2)), \n    layers.Conv2D(64,(3,3), activation=\"relu\"),\n    layers.MaxPooling2D(2,2),\n    layers.Conv2D(128,(3,3), activation=\"relu\"),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(128, (3,3), activation=\"relu\"),\n    layers.MaxPooling2D((2,2)),\n    layers.Flatten(),\n    layers.Dropout(0.5),\n    layers.Dense(512, activation=\"relu\"),\n    layers.Dense(1, activation=\"sigmoid\")\n])\n\n# Compile the model\nmodel_1.compile(loss=\"binary_crossentropy\",\n               optimizer=tf.keras.optimizers.legacy.RMSprop(learning_rate=1e-4),\n               metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:56:42.720888Z","iopub.execute_input":"2023-09-23T07:56:42.721297Z","iopub.status.idle":"2023-09-23T07:56:42.834231Z","shell.execute_reply.started":"2023-09-23T07:56:42.721264Z","shell.execute_reply":"2023-09-23T07:56:42.832868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the Convnet using data-augmentation generators","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1/255.,\n                             rotation_range=40,\n                             width_shift_range=0.2,\n                             height_shift_range=0.2,\n                             shear_range=0.2,\n                             zoom_range=0.2,\n                             horizontal_flip=True)\nvalid_datagen = ImageDataGenerator(rescale=1/255.) # Note that validation or test data should not be augmented!\ntest_datagen = ImageDataGenerator(rescale=1/255.) \n\ntrain_generator= train_datagen.flow_from_directory(train_dir,  # Target dir\n                                                  target_size=(150,150),#Resizes all images to 150 * 150\n                                                  batch_size=32,\n                                                  class_mode=\"binary\") # because we'r using binary_crossentropy loss, we need binary labels\n\nvalidation_generator = valid_datagen.flow_from_directory(validation_dir,\n                                                       target_size=(150,150),\n                                                       batch_size=20,\n                                                       class_mode=\"binary\")\n\ntest_generator = test_datagen.flow_from_directory(test_dir,\n                                              target_size=(150,150),\n                                              batch_size=20,\n                                              class_mode=\"binary\",\n                                              shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:14.653361Z","iopub.execute_input":"2023-09-23T07:31:14.653721Z","iopub.status.idle":"2023-09-23T07:31:14.804451Z","shell.execute_reply.started":"2023-09-23T07:31:14.653689Z","shell.execute_reply":"2023-09-23T07:31:14.803410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:31:14.806255Z","iopub.execute_input":"2023-09-23T07:31:14.806638Z","iopub.status.idle":"2023-09-23T07:31:14.840127Z","shell.execute_reply.started":"2023-09-23T07:31:14.806579Z","shell.execute_reply":"2023-09-23T07:31:14.839272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_1 = model_1.fit(train_generator,\n                        steps_per_epoch=len(train_generator),\n                        epochs=30,\n                       validation_data=validation_generator,\n                       validation_steps=len(validation_generator),\n                       verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T07:56:48.094124Z","iopub.execute_input":"2023-09-23T07:56:48.094536Z","iopub.status.idle":"2023-09-23T08:06:34.638384Z","shell.execute_reply.started":"2023-09-23T07:56:48.094504Z","shell.execute_reply":"2023-09-23T08:06:34.637061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nmodel_1.evaluate(test_generator)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:07:26.768916Z","iopub.execute_input":"2023-09-23T08:07:26.769298Z","iopub.status.idle":"2023-09-23T08:07:29.408733Z","shell.execute_reply.started":"2023-09-23T08:07:26.769267Z","shell.execute_reply":"2023-09-23T08:07:29.407668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracy_and_loss(history=history_1);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:07:33.163246Z","iopub.execute_input":"2023-09-23T08:07:33.163679Z","iopub.status.idle":"2023-09-23T08:07:33.917677Z","shell.execute_reply.started":"2023-09-23T08:07:33.163635Z","shell.execute_reply":"2023-09-23T08:07:33.916579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This model gives 71% accuracy which is quite good let's try some other techniques too","metadata":{}},{"cell_type":"markdown","source":"# Using a pretrained convnet\nA *pretrained network* is a saved network that was previously trained on a large dataset, typically on a large-scale image-classification task.","metadata":{}},{"cell_type":"markdown","source":"### Feature extraction\nIt consists of using the representations learned by a previous network to extract interesting features from a new samples. These features are then run through a new classifier, which is trained from scratch.","metadata":{}},{"cell_type":"markdown","source":"### Instantiating the VGG16 convolutional base","metadata":{}},{"cell_type":"code","source":"from keras.applications import VGG16\n\nconv_base = VGG16(weights= 'imagenet',\n                 include_top=False,\n                 input_shape=(150,150,3))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:09:53.634032Z","iopub.execute_input":"2023-09-23T08:09:53.634432Z","iopub.status.idle":"2023-09-23T08:09:54.589746Z","shell.execute_reply.started":"2023-09-23T08:09:53.634399Z","shell.execute_reply":"2023-09-23T08:09:54.588557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model summary\nconv_base.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:09:55.763810Z","iopub.execute_input":"2023-09-23T08:09:55.764212Z","iopub.status.idle":"2023-09-23T08:09:55.812448Z","shell.execute_reply.started":"2023-09-23T08:09:55.764178Z","shell.execute_reply":"2023-09-23T08:09:55.811633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At this point, there are two ways we could proceed:\n    \n* Runnning the convolutional  base over the dataset, recording its output to a Numpy array on disk, and then using this data as input to a standalone, densely connnected classifier. This solution is fast and cheap to run, because it only requires running the convolutional base once for every input image, and the convolutional base is by far the most expensive part of the pipeline.But for the same reason, this technique won't allow to use data augmentation.\n    \n* Extending the model we have(conv_base) by adding Dense layers on top, and runnning the whole thing end to end on the input data. This will allow us to use data augmentation, because every input image goes through the convolutional base every time it's seen by the model. But for the same reason, this technique is far more expensive than the first.\n    ","metadata":{}},{"cell_type":"markdown","source":"## Fast Feature Extraction without Data Augmentation \nWe'll start by runnning instances of the `ImageDataGenerator` to extract images as Numpy arrays as well as their labels. We'll extract features from these images by calling the `predict` method of the `conv_base` model.","metadata":{}},{"cell_type":"code","source":"%%capture\n# Extracting features using the pretrained convolutional base\nimport os\nimport numpy as np\nfrom keras.preprocessing.image import ImageDataGenerator\n\nbase_dir = \"/kaggle/working/cats_and_dogs_small\"\ntrain_dir = os.path.join(base_dir, 'train')\nvalidation_dir = os.path.join(base_dir, 'validation')\ntest_dir = os.path.join(base_dir, 'test')\n\ndatagen  = ImageDataGenerator(rescale=1/255.)\nbatch_size = 20\n\ndef extract_features(directory, sample_count):\n    features = np.zeros(shape=(sample_count,4,4,512))\n    labels = np.zeros(shape=(sample_count))\n    generator = datagen.flow_from_directory(\n                                directory,\n                                target_size=(150,150),\n                                batch_size=batch_size,\n                                class_mode=\"binary\");\n    i=0\n    for inputs_batch, labels_batch in generator:\n        features_batch = conv_base.predict(inputs_batch)\n        features[i * batch_size: (i + 1) * batch_size] = features_batch\n        labels[i * batch_size : (i +1) * batch_size] = labels_batch\n        i += 1\n        if i * batch_size >= sample_count:\n            break #Note that because generators yield data indefinitely in a loop. we must break after every image is seen once\n    return features, labels\n\ntrain_features, train_labels = extract_features(train_dir, 2000);\nvalidation_features , validation_labels = extract_features(validation_dir,1000);\ntest_features, test_labels  = extract_features(test_dir,1000);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:14:04.289924Z","iopub.execute_input":"2023-09-23T08:14:04.290329Z","iopub.status.idle":"2023-09-23T08:14:35.293776Z","shell.execute_reply.started":"2023-09-23T08:14:04.290294Z","shell.execute_reply":"2023-09-23T08:14:35.292642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The extracted features are currently of shape (sample, 4, 4, 512). We'll feed them to a densely connected classifier, so first we must flatten them to (sample,8192)","metadata":{}},{"cell_type":"code","source":"train_features = np.reshape(train_features, (2000, 4 * 4 * 512))\nvalidation_features = np.reshape(validation_features , (1000, 4 * 4 * 512))\ntest_features = np.reshape(test_features, (1000, 4 * 4 * 512))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:14:43.320689Z","iopub.execute_input":"2023-09-23T08:14:43.321087Z","iopub.status.idle":"2023-09-23T08:14:43.327485Z","shell.execute_reply.started":"2023-09-23T08:14:43.321054Z","shell.execute_reply":"2023-09-23T08:14:43.326348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining and training the densely connected classifier","metadata":{}},{"cell_type":"code","source":"from keras import models\nfrom keras import layers \nfrom keras import optimizers \n\nmodel = models.Sequential([\n    layers.Dense(256, activation=\"relu\", input_dim=4 * 4 * 512),\n    layers.Dropout(0.5),\n    layers.Dense(1, activation=\"sigmoid\")\n])\n\nmodel.compile(optimizer = optimizers.RMSprop(learning_rate=2e-5),\n             loss=\"binary_crossentropy\",\n             metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:14:45.367088Z","iopub.execute_input":"2023-09-23T08:14:45.368210Z","iopub.status.idle":"2023-09-23T08:14:45.418604Z","shell.execute_reply.started":"2023-09-23T08:14:45.368161Z","shell.execute_reply":"2023-09-23T08:14:45.417462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_features, \n                    train_labels,\n                   batch_size=20,\n                    epochs=30,\n                   validation_data=(validation_features, validation_labels))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:14:49.092746Z","iopub.execute_input":"2023-09-23T08:14:49.093146Z","iopub.status.idle":"2023-09-23T08:15:10.294267Z","shell.execute_reply.started":"2023-09-23T08:14:49.093106Z","shell.execute_reply":"2023-09-23T08:15:10.293113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nmodel.evaluate(test_features, test_labels)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:16:19.008206Z","iopub.execute_input":"2023-09-23T08:16:19.008733Z","iopub.status.idle":"2023-09-23T08:16:19.442527Z","shell.execute_reply.started":"2023-09-23T08:16:19.008685Z","shell.execute_reply":"2023-09-23T08:16:19.441480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's look at loss and accuracy curves","metadata":{}},{"cell_type":"code","source":"plot_accuracy_and_loss(history=history);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:18:23.952132Z","iopub.execute_input":"2023-09-23T08:18:23.952784Z","iopub.status.idle":"2023-09-23T08:18:24.750513Z","shell.execute_reply.started":"2023-09-23T08:18:23.952739Z","shell.execute_reply":"2023-09-23T08:18:24.749516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This model overfits after some epochs. It gives 88% accuracy on test data. You can try training the model for less epochs. Now, let the experiment going..","metadata":{}},{"cell_type":"markdown","source":"# Feature Extraction with Data Augmentation","metadata":{}},{"cell_type":"markdown","source":"  **Note:** This technique is so expensive that you should only attempt it if you have access to a GPU - it's absolutely intractable on CPU.  If you can't run your code on GPU, then the previous technique is the way to go.\n\nBecause model behave just like layers, we can add a model(like conv_base) to a `Sequential` model just like we would add a layer\n","metadata":{}},{"cell_type":"code","source":"# Adding a densely connected classifier on top of the convolutional base\nfrom keras import models\nfrom keras import layers\n\nmodel = models.Sequential([\n    conv_base,\n    layers.Flatten(),\n    layers.Dense(256, activation=\"relu\"),\n    layers.Dense(1, activation=\"sigmoid\")\n])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:22:53.461556Z","iopub.execute_input":"2023-09-23T08:22:53.462688Z","iopub.status.idle":"2023-09-23T08:22:53.567816Z","shell.execute_reply.started":"2023-09-23T08:22:53.462631Z","shell.execute_reply":"2023-09-23T08:22:53.566752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:22:55.135392Z","iopub.execute_input":"2023-09-23T08:22:55.135796Z","iopub.status.idle":"2023-09-23T08:22:55.159073Z","shell.execute_reply.started":"2023-09-23T08:22:55.135762Z","shell.execute_reply":"2023-09-23T08:22:55.158279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freezing the layers of convbase\nprint(\"This is the number of trainable weights before freezing the conv_base:\", len(model.trainable_weights))\nconv_base.trainable = False\nprint(\"This is the number of trainable weights after freezing the conv_base:\", len(model.trainable_weights))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:23:00.409642Z","iopub.execute_input":"2023-09-23T08:23:00.410347Z","iopub.status.idle":"2023-09-23T08:23:00.417478Z","shell.execute_reply.started":"2023-09-23T08:23:00.410311Z","shell.execute_reply":"2023-09-23T08:23:00.416407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With this setup, only the weights from the two `Dense` layers that we added will be trained. That's a total of four weight tensors: two per layer (the main weight matrix and the bias vector). \n\n**Note:** In order for these changes to take effect , you must first compile the model. If you ever modify weight trainability after compilation, you should then recompile the model, or these changes will be ignored.","metadata":{}},{"cell_type":"markdown","source":"### Training the model end to end with a frozen convolutional base","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers\n\ntrain_datagen = ImageDataGenerator(rescale=1/255.,\n                             rotation_range=40,\n                             width_shift_range=0.2,\n                             height_shift_range=0.2,\n                             shear_range=0.2,\n                             zoom_range=0.2,\n                             horizontal_flip=True,\n                             fill_mode=\"nearest\")\n\nvalid_datagen = ImageDataGenerator(rescale=1/255.) # Note that validation or test data should not be augmented!\ntest_datagen = ImageDataGenerator(rescale=1/255.) \n\ntrain_generator= train_datagen.flow_from_directory(train_dir,  # Target dir\n                                                  target_size=(150,150),#Resizes all images to 150 * 150\n                                                  batch_size=32,\n                                                  class_mode=\"binary\") # because we'r using binary_crossentropy loss, we need binary labels\n\nvalidation_generator = valid_datagen.flow_from_directory(validation_dir,\n                                                       target_size=(150,150),\n                                                       batch_size=20,\n                                                       class_mode=\"binary\")\n\ntest_generator = test_datagen.flow_from_directory(test_dir,\n                                              target_size=(150,150),\n                                              batch_size=20,\n                                              class_mode=\"binary\",\n                                              shuffle=False)\n\nmodel.compile(loss=\"binary_crossentropy\",\n              optimizer=optimizers.RMSprop(learning_rate=2e-5),\n              metrics=[\"accuracy\"])\n\nhistory = model.fit(train_generator,\n                   steps_per_epoch=len(train_generator),\n                   epochs=30,\n                   validation_data = validation_generator,\n                   validation_steps=len(validation_generator))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:23:09.755691Z","iopub.execute_input":"2023-09-23T08:23:09.756092Z","iopub.status.idle":"2023-09-23T08:38:20.797451Z","shell.execute_reply.started":"2023-09-23T08:23:09.756059Z","shell.execute_reply":"2023-09-23T08:38:20.796121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nmodel.evaluate(test_generator)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:39:11.992961Z","iopub.execute_input":"2023-09-23T08:39:11.993405Z","iopub.status.idle":"2023-09-23T08:39:15.102431Z","shell.execute_reply.started":"2023-09-23T08:39:11.993369Z","shell.execute_reply":"2023-09-23T08:39:15.101399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracy_and_loss(history=history);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:39:17.703221Z","iopub.execute_input":"2023-09-23T08:39:17.703642Z","iopub.status.idle":"2023-09-23T08:39:18.428803Z","shell.execute_reply.started":"2023-09-23T08:39:17.703581Z","shell.execute_reply":"2023-09-23T08:39:18.426243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is much better then what we achieved from training Convnet from scratch","metadata":{}},{"cell_type":"markdown","source":"# Fine-tuning\nIt consists of unfreezing a few of the top layers of a frozen model base used for feature extraction, and jointly training both the newly added part of the model(in this case, the fully connected classifier) and these top layers. This is called *fine-tuning* because it slightly adjusts the more abstract representation of the model being rescued, in order to make them more relevant for the problem at hand. \n\nIf the classifier isn't already trained, then the erroe signal propagating through the network during training will be too large, and the representation previously learned by the layers being fine-tuned will be destroyed. Thus the steps for fine-tuning a network are as follow:\n    1. Add custom network on top of an already-trained base network\n    2. Freeze the base network.\n    3. Train the part which is added.\n    4. Unfreeze some layers in the base network.\n    5. jointly train both these layers and the part added.\n\nWe have already completed first three steps when doing feature extraction. let's process with step-4...  ","metadata":{}},{"cell_type":"code","source":"# As a remainder, this is what conv base looks like:\nconv_base.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:39:30.640272Z","iopub.execute_input":"2023-09-23T08:39:30.640681Z","iopub.status.idle":"2023-09-23T08:39:30.679659Z","shell.execute_reply.started":"2023-09-23T08:39:30.640648Z","shell.execute_reply":"2023-09-23T08:39:30.678788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll fine tune last three convolutional layers,which means all layers upto `block4_pool` should be frozen.\n\n**Note:** You need to consider the following while fine-tuning:\n* Earlier layers in the conv base encode more generic, reusable features, whereas layers higher up encode more-specialized features.It's more useful to fine-tune the more specialized featurees, because these are the ones that need to be repurposed on your new problem. There would be fast-decreasing returns in fine-tuning lower layers.\n\n* The more parameters you're training, the more your're at risk of overfitting. The convolutional base has 15 million parameters, so it would be risky to attempt to train it on your small dataset.\n\nThus, in this situation, it's a good strategy to fine-tune only the top two or three layers in the convolutional base,","metadata":{}},{"cell_type":"code","source":"# Freezing all layers up to a specific one\nconv_base.trainable = True\n\nfor layer in conv_base.layers[:-4]:\n    layer.trainable = False\n\n# Check layers\nfor i, layer in enumerate(model.layers[0].layers):\n    print(i, layer.name, layer.trainable)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:39:37.309830Z","iopub.execute_input":"2023-09-23T08:39:37.310268Z","iopub.status.idle":"2023-09-23T08:39:37.319735Z","shell.execute_reply.started":"2023-09-23T08:39:37.310235Z","shell.execute_reply":"2023-09-23T08:39:37.318460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine-tuning the model \nmodel.compile(loss=\"binary_crossentropy\",\n             optimizer=optimizers.RMSprop(learning_rate=1e-5),\n             metrics=[\"accuracy\"])\n\nhistory = model.fit(train_generator,\n                    steps_per_epoch=len(train_generator),\n                   epochs=30,\n                   validation_data=validation_generator,\n                   validation_steps=50)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:39:59.139031Z","iopub.execute_input":"2023-09-23T08:39:59.139421Z","iopub.status.idle":"2023-09-23T08:55:36.692175Z","shell.execute_reply.started":"2023-09-23T08:39:59.139388Z","shell.execute_reply":"2023-09-23T08:55:36.691004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the loss and accuracy\n# Plotting the results\nimport matplotlib.pyplot as plt\n\nacc = history.history[\"accuracy\"]\nval_acc = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\n\nepochs = range(1, len(loss) +1)\n\n# plot loss curve\nplt.plot(epochs, loss, \"bo\", label=\"Training loss\")\nplt.plot(epochs, val_loss , \"r\", label=\"Validation loss\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"loss\")\nplt.title(\"Training and Validation loss\")\nplt.legend()\n\nplt.figure()\n\n# plot accuracy curve\nplt.plot(epochs, acc, \"bo\", label=\"Training accuracy\")\nplt.plot(epochs, val_acc , \"r\", label=\"Validation accuracy\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"accuracy\")\nplt.title(\"Training and Validation accuracy\")\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:57:39.462384Z","iopub.execute_input":"2023-09-23T08:57:39.462791Z","iopub.status.idle":"2023-09-23T08:57:40.302260Z","shell.execute_reply.started":"2023-09-23T08:57:39.462758Z","shell.execute_reply":"2023-09-23T08:57:40.301032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These curves looks noisy . To make them readable, we can smooth them by replacing every loss and accuracy with exponential moving averages of these quantities. Let's make a function for it.. ","metadata":{}},{"cell_type":"markdown","source":"## Smoothing the plots","metadata":{}},{"cell_type":"code","source":"def smooth_curve(points, factor=0.8):\n    smoothed_points= []\n    for point in points:\n        if smoothed_points:\n            previous = smoothed_points[-1]\n            smoothed_points.append(previous * factor + point * (1- factor))\n        else:\n            smoothed_points.append(point)\n    return smoothed_points\n\n\nacc = history.history[\"accuracy\"]\nval_acc = history.history[\"val_accuracy\"]\nloss = history.history[\"loss\"]\nval_loss = history.history[\"val_loss\"]\n\nplt.plot(epochs, \n         smooth_curve(acc), 'bo', label=\"Smoothed training acc\")\nplt.plot(epochs,\n        smooth_curve(val_acc), 'r' , label=\"Smoothed validation acc\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"accuracy\")\nplt.title(\"Training and Validation Accuracy\")\nplt.legend()\n\nplt.figure()\n\n# plot loss curve\nplt.plot(epochs, \n         smooth_curve(loss), 'bo', label=\"Smoothed training loss\")\nplt.plot(epochs,\n        smooth_curve(val_loss), 'r' , label=\"Smoothed validation loss\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"loss\")\nplt.title(\"Training and Validation Loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:57:50.449761Z","iopub.execute_input":"2023-09-23T08:57:50.450159Z","iopub.status.idle":"2023-09-23T08:57:51.121546Z","shell.execute_reply.started":"2023-09-23T08:57:50.450127Z","shell.execute_reply":"2023-09-23T08:57:51.120549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving the model\nmodel.save(\"cats_and_dogs_small_2.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:57:58.720680Z","iopub.execute_input":"2023-09-23T08:57:58.721053Z","iopub.status.idle":"2023-09-23T08:57:58.927724Z","shell.execute_reply.started":"2023-09-23T08:57:58.721022Z","shell.execute_reply":"2023-09-23T08:57:58.926645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_generator)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:58:00.065702Z","iopub.execute_input":"2023-09-23T08:58:00.068451Z","iopub.status.idle":"2023-09-23T08:58:05.265925Z","shell.execute_reply.started":"2023-09-23T08:58:00.068413Z","shell.execute_reply":"2023-09-23T08:58:05.264610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Woah! We got the highest accuracy with this model!","metadata":{}},{"cell_type":"markdown","source":"# Visualizing what convnets learn\nIt's often said that deep-learning models are \"black boxes\": learning representations that are difficult to extract and present in a human-readable form. Although this is partially true for certain types of deep-learning models, it is defintely not true for Convnets. The representations learned by convnets are highly amenable to visualization, in large oart because they are *representations of visual concepts*. The three most accesible and useful ones are:\n* *Visualizing intermediate convnet outputs(intermediate activations)* - Useful for understanding how successive convnet layers transform their input, and for getting a first idea of the meaning of individual convnet filters\n* *Visualizing convnets filters* - Useful for understanding precisely what visual pattern or concept each filter in a convnet is receptive to.\n* *Visualizing heatmaps of class activation in an image* - Useful for understanding which parts of an image were identified as belonging to a given class, thus allowing you to localize objects in images.","metadata":{}},{"cell_type":"markdown","source":"## Visualizing intermediate activations\nIt consists of displaying the feature maps that are output by various convolution and pooling layers in a network, given a certain input(the output of layer is often called its *activation*, the output of the activation function). ","metadata":{}},{"cell_type":"code","source":"# Let's start by loading the model \nfrom keras.models import load_model\nmodel = load_model(\"/kaggle/input/model/cats_and_dogs_small_1.h5\")\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:58:48.764749Z","iopub.execute_input":"2023-09-23T08:58:48.765133Z","iopub.status.idle":"2023-09-23T08:58:49.217993Z","shell.execute_reply.started":"2023-09-23T08:58:48.765101Z","shell.execute_reply":"2023-09-23T08:58:49.217108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we'll get an input picture from test data ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.utils import load_img # Preprocess the image into a 4D tensor\nimport numpy as np\n\nimg_path = \"/kaggle/working/cats_and_dogs_small/test/cats/cat.1700.jpg\"\n\n# Load the image into a PIL Image instance.\nimg = load_img(img_path, target_size=(150, 150))\n\n# Convert the PIL Image instance to a NumPy array.\nimg_tensor = np.array(img)\n\n# Expand the dimensions of the tensor.\nimg_tensor = np.expand_dims(img_tensor, axis=0)\nimg_tensor = img_tensor.astype(np.float32)\nimg_tensor /= 255.\n\nprint(img_tensor.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:58:56.263727Z","iopub.execute_input":"2023-09-23T08:58:56.264109Z","iopub.status.idle":"2023-09-23T08:58:56.274615Z","shell.execute_reply.started":"2023-09-23T08:58:56.264078Z","shell.execute_reply":"2023-09-23T08:58:56.273435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying the test picture\nimport matplotlib.pyplot as plt\n\nplt.imshow(img_tensor[0])\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:59:02.429345Z","iopub.execute_input":"2023-09-23T08:59:02.429775Z","iopub.status.idle":"2023-09-23T08:59:02.671326Z","shell.execute_reply.started":"2023-09-23T08:59:02.429740Z","shell.execute_reply":"2023-09-23T08:59:02.669640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Instantiating a model from an input tensor and a list of output tensors","metadata":{}},{"cell_type":"code","source":"from keras import models\n\nlayers_outputs = [layer.output for layer in model.layers[:8]] # Extracts the outputs of the top eight layers\nactivation_model  = models.Model(inputs=model.input, outputs=layers_outputs)# Create a model that will return these outputs , given the model input","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:59:08.214751Z","iopub.execute_input":"2023-09-23T08:59:08.215130Z","iopub.status.idle":"2023-09-23T08:59:08.226958Z","shell.execute_reply.started":"2023-09-23T08:59:08.215099Z","shell.execute_reply":"2023-09-23T08:59:08.225903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Running the model in predict mode\nactivations = activation_model.predict(img_tensor) #Returns a list of five Numpy arrays: one per layer activation","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:59:11.067107Z","iopub.execute_input":"2023-09-23T08:59:11.067505Z","iopub.status.idle":"2023-09-23T08:59:11.380984Z","shell.execute_reply.started":"2023-09-23T08:59:11.067473Z","shell.execute_reply":"2023-09-23T08:59:11.379857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For instance, this is the activation of the first convolution layer for the cat image input:","metadata":{}},{"cell_type":"code","source":"first_layer_activation = activations[0]\nprint(first_layer_activation.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T08:59:14.027418Z","iopub.execute_input":"2023-09-23T08:59:14.027796Z","iopub.status.idle":"2023-09-23T08:59:14.033658Z","shell.execute_reply.started":"2023-09-23T08:59:14.027764Z","shell.execute_reply":"2023-09-23T08:59:14.032430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the fourth channel\nimport matplotlib.pyplot as plt\nplt.matshow(first_layer_activation[0, :, :, 4], cmap = 'viridis');","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:01:20.214445Z","iopub.execute_input":"2023-09-23T09:01:20.215631Z","iopub.status.idle":"2023-09-23T09:01:20.736608Z","shell.execute_reply.started":"2023-09-23T09:01:20.215562Z","shell.execute_reply":"2023-09-23T09:01:20.735620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:** Your own channels may vary, because the specific filters learned by convolutional layers aren't deterministic.","metadata":{}},{"cell_type":"code","source":"# Visualizing the seventh channel\nplt.matshow(first_layer_activation[0, :, :, 7], cmap = 'viridis');","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:01:34.972250Z","iopub.execute_input":"2023-09-23T09:01:34.972790Z","iopub.status.idle":"2023-09-23T09:01:35.514392Z","shell.execute_reply.started":"2023-09-23T09:01:34.972749Z","shell.execute_reply":"2023-09-23T09:01:35.513325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing every channel in every intermediate activation","metadata":{}},{"cell_type":"code","source":"layer_names = []\nfor layer in model.layers[:8]:\n    layer_names.append(layer.name) #Names of the layers, so we can have them as part of plot\n\nimages_per_row= 16\n\nfor layer_name, layer_activation in zip(layer_names, activations): #Display the feature map\n    n_features  = layer_activation.shape[-1] # number of features in the feature map\n    \n    size = layer_activation.shape[1] #feature map has shape (I, size, size, n_features)\n    \n    n_cols = n_features // images_per_row #Tiles the activation in the matrix\n    display_grid = np.zeros((size * n_cols, images_per_row * size))\n    \n    for col in range(n_cols): #Tiles each filter into a big horizontal grid\n        for row in range(images_per_row):\n            channel_image = layer_activation[0,:,:, col * images_per_row + row]\n            channel_image -= channel_image.mean() #Post-preprocess the feature to make it visually palatable\n            channel_image /= 64\n            channel_image += 128\n            channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n            display_grid[col * size : (col +1) * size , #Display the grid\n                        row  * size : (row+1) * size] = channel_image\n    scale = 1./size\n    plt.figure(figsize = (scale * display_grid.shape[1],\n                         scale * display_grid.shape[0]))\n    plt.title(layer_name)\n    plt.grid(False)\n    plt.imshow(display_grid, aspect='auto', cmap ='viridis')","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:01:40.422510Z","iopub.execute_input":"2023-09-23T09:01:40.424937Z","iopub.status.idle":"2023-09-23T09:01:44.371070Z","shell.execute_reply.started":"2023-09-23T09:01:40.424890Z","shell.execute_reply":"2023-09-23T09:01:44.369924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing convnet filters \nAnother easy way to inspect the filters learned by convnets is to display the visual pattern that each filter is meant to respond to. This can be done with `gradient ascent in inout space:` applying `gradient descent` to the value of the input image of convnet so as to `maximize` the response of a specific filter, starting from the blank input image. The resulting input image will be one that the chosen filter is maximally responsive to. ","metadata":{}},{"cell_type":"markdown","source":"### Defining the loss tensor of filter visualization","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\n# Disable eager execution\ntf.compat.v1.disable_eager_execution()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:01:58.310548Z","iopub.execute_input":"2023-09-23T09:01:58.310954Z","iopub.status.idle":"2023-09-23T09:01:58.315843Z","shell.execute_reply.started":"2023-09-23T09:01:58.310922Z","shell.execute_reply":"2023-09-23T09:01:58.314768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications import VGG16\nfrom keras import backend as K\n\nmodel = VGG16(weights=\"imagenet\",\n             include_top=False)\n\nlayer_name = 'block3_conv1'\nfilter_index = 0\n\nlayer_output = model.get_layer(layer_name).output\nloss = K.mean(layer_output[:,:,:,filter_index])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:01:59.156488Z","iopub.execute_input":"2023-09-23T09:01:59.157257Z","iopub.status.idle":"2023-09-23T09:02:00.184382Z","shell.execute_reply.started":"2023-09-23T09:01:59.157221Z","shell.execute_reply":"2023-09-23T09:02:00.183305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Obtaining the gradient of the loss with regard to the input","metadata":{}},{"cell_type":"code","source":"grads = K.gradients(loss, model.input)[0] # call to gradients returns a list of tensors(of size 1 in this case). Hence, we keep only first element- which is a tensor","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:02.693673Z","iopub.execute_input":"2023-09-23T09:02:02.694816Z","iopub.status.idle":"2023-09-23T09:02:02.747834Z","shell.execute_reply.started":"2023-09-23T09:02:02.694770Z","shell.execute_reply":"2023-09-23T09:02:02.746751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gradient normalization trick","metadata":{"execution":{"iopub.status.busy":"2023-07-23T13:42:55.289881Z","iopub.execute_input":"2023-07-23T13:42:55.290318Z","iopub.status.idle":"2023-07-23T13:42:55.300649Z","shell.execute_reply.started":"2023-07-23T13:42:55.290271Z","shell.execute_reply":"2023-07-23T13:42:55.299603Z"}}},{"cell_type":"code","source":"grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) #Add 1e-5 before diving to avoid accidently dividing by 0","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:06.296052Z","iopub.execute_input":"2023-09-23T09:02:06.296482Z","iopub.status.idle":"2023-09-23T09:02:06.310503Z","shell.execute_reply.started":"2023-09-23T09:02:06.296450Z","shell.execute_reply":"2023-09-23T09:02:06.309348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fetching Numpy Output values given Numpy input values","metadata":{}},{"cell_type":"code","source":"iterate = K.function([model.input], [loss, grads])\n\nimport numpy as np\nloss_value, grads_value = iterate([np.zeros((1,150,150,3))])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:08.875285Z","iopub.execute_input":"2023-09-23T09:02:08.878175Z","iopub.status.idle":"2023-09-23T09:02:09.500209Z","shell.execute_reply.started":"2023-09-23T09:02:08.878128Z","shell.execute_reply":"2023-09-23T09:02:09.498809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loss maximization via stochastic gradient descent ","metadata":{}},{"cell_type":"code","source":"input_img_data = np.random.random((1,150,150,3)) * 20 + 128 # Starts from a gray image with some noise\n\nstep = 1 # magnitude of each gradient update\nfor i in range(40): # Runs gradient ascent for 40 steps\n    loss_value, grads_value = iterate([input_img_data]) # computes the loss value and gradient value\n    input_img_data += grads_value * step #adjusts the input image in the direction that maximizes the loss ","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:12.348935Z","iopub.execute_input":"2023-09-23T09:02:12.349343Z","iopub.status.idle":"2023-09-23T09:02:12.586390Z","shell.execute_reply.started":"2023-09-23T09:02:12.349303Z","shell.execute_reply":"2023-09-23T09:02:12.585338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utility function to convert a tensor into a valid image","metadata":{}},{"cell_type":"code","source":"def deprocess_image(x):\n    #Normalizes the tensor:centers on 0, ensures that std is 0.1\n    x -= x.mean()\n    x /= (x.std() + 1e-5)\n    x *= 0.1 \n    \n    #clips to [0,1]\n    x += 0.5\n    x = np.clip(x,0,1)\n    \n    #converts to an RGB array\n    x *= 255\n    x = np.clip(x, 0, 255).astype('uint8') \n    return x ","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:14.680467Z","iopub.execute_input":"2023-09-23T09:02:14.681282Z","iopub.status.idle":"2023-09-23T09:02:14.688195Z","shell.execute_reply.started":"2023-09-23T09:02:14.681243Z","shell.execute_reply":"2023-09-23T09:02:14.686808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function to generate filter visualization","metadata":{}},{"cell_type":"code","source":"def generate_pattern(layer_name, filter_index, size=150):\n    layer_output = model.get_layer(layer_name).output #builds s loss function thst maximizes activation of nth filter of layer under consideration\n    loss = K.mean(layer_output[:, :, :, filter_index])\n    \n    grads = K.gradients(loss, model.input)[0] #computes the gradient of the input picture with regard to this loss\n    \n    grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) #normalization gradient\n    \n    iterate = K.function([model.input], [loss, grads])\n    input_img_data = np.random.random((1,size,size,3)) * 20 + 128 # Starts from a gray image with some noise\n    step = 1 # magnitude of each gradient update\n    for i in range(40): # Runs gradient ascent for 40 steps\n        loss_value, grads_value = iterate([input_img_data]) # computes the loss value and gradient value\n        input_img_data += grads_value * step #adjusts the input image in the direction that maximizes the loss \n    \n    img = input_img_data[0]\n    return deprocess_image(img)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:17.613734Z","iopub.execute_input":"2023-09-23T09:02:17.614201Z","iopub.status.idle":"2023-09-23T09:02:17.626544Z","shell.execute_reply.started":"2023-09-23T09:02:17.614167Z","shell.execute_reply":"2023-09-23T09:02:17.625435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.imshow(generate_pattern('block3_conv1', 0))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:18.933445Z","iopub.execute_input":"2023-09-23T09:02:18.934673Z","iopub.status.idle":"2023-09-23T09:02:19.711869Z","shell.execute_reply.started":"2023-09-23T09:02:18.934581Z","shell.execute_reply":"2023-09-23T09:02:19.710836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generating a grid of all filter response patterns in a layer","metadata":{}},{"cell_type":"code","source":"layer_name = 'block1_conv1'\nsize= 64\nmargin = 5\n\nresults =  np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3)) # Empty(black) image to store results\n\nfor i in range(8): #iterates over the rows of the results grid\n    for j in range(8): # iterates for the columns of the results grid\n        filter_img =  generate_pattern(layer_name, i + (j*8), size) # Generates the pattern for filter i + (j * 8) in layer_name\n        # puts the result in the square (i,j) of the results grid\n        horizontal_start = i * size + i * margin\n        horizontal_end = horizontal_start + size\n        vertical_start = j * size + j * margin \n        vertical_end = vertical_start + size\n        results[horizontal_start : horizontal_end,\n               vertical_start : vertical_end, :] = filter_img\n\n# Displays the results grid\nplt.figure(figsize= (20,20))\nplt.imshow(results)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:30.707231Z","iopub.execute_input":"2023-09-23T09:02:30.707629Z","iopub.status.idle":"2023-09-23T09:02:42.883636Z","shell.execute_reply.started":"2023-09-23T09:02:30.707574Z","shell.execute_reply":"2023-09-23T09:02:42.882535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for the filter pattern in block2_conv1\nlayer_name = 'block2_conv1'\nsize= 64\nmargin = 5\n\nresults =  np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3)) # Empty(black) image to store results\n\nfor i in range(8): #iterates over the rows of the results grid\n    for j in range(8): # iterates for the columns of the results grid\n        filter_img =  generate_pattern(layer_name, i + (j*8), size) # Generates the pattern for filter i + (j * 8) in layer_name\n        # puts the result in the square (i,j) of the results grid\n        horizontal_start = i * size + i * margin\n        horizontal_end = horizontal_start + size\n        vertical_start = j * size + j * margin \n        vertical_end = vertical_start + size\n        results[horizontal_start : horizontal_end,\n               vertical_start : vertical_end, :] = filter_img\n\n# Displays the results grid\nplt.figure(figsize= (20,20))\nplt.imshow(results)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:02:42.886204Z","iopub.execute_input":"2023-09-23T09:02:42.886622Z","iopub.status.idle":"2023-09-23T09:03:05.547996Z","shell.execute_reply.started":"2023-09-23T09:02:42.886570Z","shell.execute_reply":"2023-09-23T09:03:05.546788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for the filter pattern in block3_conv1\nlayer_name = 'block3_conv1'\nsize= 64\nmargin = 5\n\nresults =  np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3)) # Empty(black) image to store results\n\nfor i in range(8): #iterates over the rows of the results grid\n    for j in range(8): # iterates for the columns of the results grid\n        filter_img =  generate_pattern(layer_name, i + (j*8), size) # Generates the pattern for filter i + (j * 8) in layer_name\n        # puts the result in the square (i,j) of the results grid\n        horizontal_start = i * size + i * margin\n        horizontal_end = horizontal_start + size\n        vertical_start = j * size + j * margin \n        vertical_end = vertical_start + size\n        results[horizontal_start : horizontal_end,\n               vertical_start : vertical_end, :] = filter_img\n\n# Displays the results grid\nplt.figure(figsize= (20,20))\nplt.imshow(results);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:03:05.549681Z","iopub.execute_input":"2023-09-23T09:03:05.550156Z","iopub.status.idle":"2023-09-23T09:03:41.961550Z","shell.execute_reply.started":"2023-09-23T09:03:05.550120Z","shell.execute_reply":"2023-09-23T09:03:41.960120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for the filter pattern in block4_conv1\nlayer_name = 'block4_conv1'\nsize= 64\nmargin = 5\n\nresults =  np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3)) # Empty(black) image to store results\n\nfor i in range(8): #iterates over the rows of the results grid\n    for j in range(8): # iterates for the columns of the results grid\n        filter_img =  generate_pattern(layer_name, i + (j*8), size) # Generates the pattern for filter i + (j * 8) in layer_name\n        # puts the result in the square (i,j) of the results grid\n        horizontal_start = i * size + i * margin\n        horizontal_end = horizontal_start + size\n        vertical_start = j * size + j * margin \n        vertical_end = vertical_start + size\n        results[horizontal_start : horizontal_end,\n               vertical_start : vertical_end, :] = filter_img\n\n# Displays the results grid\nplt.figure(figsize= (20,20))\nplt.imshow(results);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:03:41.964242Z","iopub.execute_input":"2023-09-23T09:03:41.964682Z","iopub.status.idle":"2023-09-23T09:04:40.372093Z","shell.execute_reply.started":"2023-09-23T09:03:41.964640Z","shell.execute_reply":"2023-09-23T09:04:40.370829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing heatmaps of class activation\nThis is useful for understanding which parts of a given image lead a convnet to its final classification decision. This is helpful for debugging the decision process of a convnet, particularly in the case of classification mistake. It also allows to loacte specific objects in an image.\n\nThis general category of techniques is called `classs activation map(CAM)` visualization, and it consists of producing heatmaps of class activation over input images. A class activation heatmap is a 2D grid of scores associated with a specific output class, computed for every location in any input image, indicating how important each location is with respect to the class under consideration. ","metadata":{}},{"cell_type":"markdown","source":"### Loading the VGG16 network with pretrained weights ","metadata":{"execution":{"iopub.status.busy":"2023-07-24T13:32:48.344917Z","iopub.execute_input":"2023-07-24T13:32:48.345322Z","iopub.status.idle":"2023-07-24T13:32:48.350415Z","shell.execute_reply.started":"2023-07-24T13:32:48.345293Z","shell.execute_reply":"2023-07-24T13:32:48.349066Z"}}},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16\n\nmodel = VGG16(weights='imagenet') #Note: we included the densely connected classifer on top; in all previous cases, we discarded it","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:05:34.276399Z","iopub.execute_input":"2023-09-23T09:05:34.276819Z","iopub.status.idle":"2023-09-23T09:05:46.183897Z","shell.execute_reply.started":"2023-09-23T09:05:34.276785Z","shell.execute_reply":"2023-09-23T09:05:46.182744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing an input image for VGG16","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing import image\nfrom keras.applications.vgg16 import preprocess_input, decode_predictions\nimport numpy as np \nimg_path = \"/kaggle/input/elephantimage/elephant_1.jpeg\"\n\nimg = image.load_img(img_path, target_size=(224,224)) #PythonImaginglibrary(PIL) image of size 224,224\n\nx = image.img_to_array(img)\n\nx = np.expand_dims(x, axis=0) #Add dimension to transform array into batch of size(1,224,224,3)\n\nx = preprocess_input(x) # Preprocesses the batch(this does channel-wise color normalization)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:06:02.033148Z","iopub.execute_input":"2023-09-23T09:06:02.033545Z","iopub.status.idle":"2023-09-23T09:06:02.063102Z","shell.execute_reply.started":"2023-09-23T09:06:02.033502Z","shell.execute_reply":"2023-09-23T09:06:02.062060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the image\nimport matplotlib.pyplot as plt\nplt.axis(False)\nplt.imshow(img);","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:06:04.771284Z","iopub.execute_input":"2023-09-23T09:06:04.771818Z","iopub.status.idle":"2023-09-23T09:06:05.030730Z","shell.execute_reply.started":"2023-09-23T09:06:04.771777Z","shell.execute_reply":"2023-09-23T09:06:05.029657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run the pretrained network on the image and decode its prediction vector back to human-readable format\npreds = model.predict(x)\nprint(\"Predicted\", decode_predictions(preds, top=3)[0])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:06:07.852994Z","iopub.execute_input":"2023-09-23T09:06:07.853787Z","iopub.status.idle":"2023-09-23T09:06:09.194558Z","shell.execute_reply.started":"2023-09-23T09:06:07.853750Z","shell.execute_reply":"2023-09-23T09:06:09.193279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.argmax(preds[0])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T09:07:04.467763Z","iopub.execute_input":"2023-09-23T09:07:04.468181Z","iopub.status.idle":"2023-09-23T09:07:04.475679Z","shell.execute_reply.started":"2023-09-23T09:07:04.468149Z","shell.execute_reply":"2023-09-23T09:07:04.474266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To visualize which part of the image are the most elephant-like, let's setup `GRAD-CAM: Visual Explanation from Deep Networks via Gradient-based Localization` process","metadata":{}},{"cell_type":"markdown","source":"### Setting up the GRAD-CAM algorithm","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\n\nelephant_output = model.output[:, 101] #elephant entry in prediction vector\nlast_conv_layer = model.get_layer(\"block5_conv3\") #output feature map of the block5_conv3_layer\n\n# Gradient of african elephant class with regard to the ouput feature map of block5_conv3\ngrads = K.gradients(elephant_output, last_conv_layer.output)[0]\n\n# Vector of shape(512,), where each entry is the mean intensity of the gradient over a specific feature-map channel\npooled_grads = K.mean(grads, axis=(0,1,2))\n\n# access the values of quantities we just defined:pooled_grads and the output feature map of block5_conv3, given a sample image\niterate = K.function([model.input],\n                    [pooled_grads, last_conv_layer.output[0]])\n\n# Values of these two quantities, as Numpy arrays, given sample image of two elephant\npooled_grads_value, conv_layer_output_value = iterate([x])\n\n# Multiples each channel in the feature-map array by \"how important this channel is\" with regard to the \"elephant\" class\nfor i in range(512):\n    conv_layer_output_value[:, :, i] *= pooled_grads_value[i]\n\n# channel-wise mean of the resulting feature map is the heatmap of the class activation\nheatmap = np.mean(conv_layer_output_value, axis=-1)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T12:10:33.883019Z","iopub.execute_input":"2023-07-25T12:10:33.889073Z","iopub.status.idle":"2023-07-25T12:10:34.724013Z","shell.execute_reply.started":"2023-07-25T12:10:33.889027Z","shell.execute_reply":"2023-07-25T12:10:34.722912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Heatmap post-processing","metadata":{}},{"cell_type":"code","source":"heatmap = np.maximum(heatmap, 0)\nheatmap /= np.max(heatmap) #normalize the heatmap between 0 and 1\nplt.matshow(heatmap)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T12:10:40.181753Z","iopub.execute_input":"2023-07-25T12:10:40.182186Z","iopub.status.idle":"2023-07-25T12:10:40.633807Z","shell.execute_reply.started":"2023-07-25T12:10:40.182130Z","shell.execute_reply":"2023-07-25T12:10:40.632712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Superimposing the heatmap on original image","metadata":{}},{"cell_type":"code","source":"import cv2\n\nimg = cv2.imread(img_path)\nheatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n\nheatmap = np.uint8(255 * heatmap)\n\nheatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)\n\nsuperimposed_img = heatmap * 0.4 + img\n\ncv2.imwrite(\"/kaggle/working/elephant_cam.jpg\", superimposed_img)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T12:13:37.109968Z","iopub.execute_input":"2023-07-25T12:13:37.110390Z","iopub.status.idle":"2023-07-25T12:13:37.130360Z","shell.execute_reply.started":"2023-07-25T12:13:37.110356Z","shell.execute_reply":"2023-07-25T12:13:37.129158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img= \"/kaggle/working/elephant_cam.jpg\"\nimg = image.load_img(img, target_size=(224,224)) #PythonImaginglibrary(PIL) image of size 224,224\n\nplt.axis(False)\nplt.imshow(img);","metadata":{"execution":{"iopub.status.busy":"2023-07-25T12:17:23.970271Z","iopub.execute_input":"2023-07-25T12:17:23.970632Z","iopub.status.idle":"2023-07-25T12:17:24.147717Z","shell.execute_reply.started":"2023-07-25T12:17:23.970600Z","shell.execute_reply":"2023-07-25T12:17:24.146752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This Visulalization technique answers two important questions:\n* Why did the network think this image contained an African elephant?\n* Where is the african elephant located in the picture?","metadata":{}}]}