In this work, I will create a convolutional neural network to classify images of cats and dogs, using the cat and dog database in kaggle.
\n \nFirst, a model will be created from scratch and the outputs and their metrics will be analyzed. Then the knowledge transfer method will be used, using the convolutional part of the pretrained network of Imagenet VGG16, to work several methods until the desired precision is achieved.
\n\n\n\n \n## Table of Contents \n \n1. [IMPORTING LIBRARIES](#1)\n \n2. [LOADING DATA](#2)\n \n3. [DATA PREPROCESSING](#3) \n \n4. [MODEL SELECTION AND TRAINING](#4)\n\n5. [GRANDCAM VISUALIZATION](#5)\n \n6. [EVALUATING MODELS](#6)\n\n \n \n\n# 1. Importing Libraries","metadata":{"execution":{"iopub.status.busy":"2023-04-11T02:27:49.213385Z","iopub.execute_input":"2023-04-11T02:27:49.214112Z","iopub.status.idle":"2023-04-11T02:27:49.326608Z","shell.execute_reply.started":"2023-04-11T02:27:49.214073Z","shell.execute_reply":"2023-04-11T02:27:49.325626Z"}}},{"cell_type":"code","source":"import os, shutil, sys\nimport cv2\nimport skimage.io as io\nfrom skimage.transform import resize\nfrom PIL import Image\n\nimport numpy as np\nimport pandas as pd\nfrom contextlib import contextmanager\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import Nadam, RMSprop\nfrom keras.applications import VGG16 \nfrom keras.applications.vgg16 import preprocess_input \n\nfrom sklearn.metrics import confusion_matrix, roc_auc_score\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom keras.callbacks import TensorBoard","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:20:54.354918Z","iopub.execute_input":"2023-04-15T07:20:54.355623Z","iopub.status.idle":"2023-04-15T07:21:03.584921Z","shell.execute_reply.started":"2023-04-15T07:20:54.355559Z","shell.execute_reply":"2023-04-15T07:21:03.583847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 2. Loading Data\n\nWe will start by importing the data and ordering it appropriately to be able to use a keras generator
\n\n","metadata":{}},{"cell_type":"code","source":"try:\n base_dir = '/kaggle/working'\n os.stat(base_dir)\n\n # Directorios de entrenamiento\n # validacion y evaluacion \n os.mkdir(\"/kaggle/working/train\")\n train_dir = os.path.join(base_dir, 'train')\n os.stat(train_dir)\n os.mkdir(\"/kaggle/working/validation\")\n validation_dir = os.path.join(base_dir, 'validation')\n os.stat(validation_dir)\n os.mkdir(\"/kaggle/working/test\")\n test_dir = os.path.join(base_dir, 'test')\n os.stat(test_dir)\n\n # Directorio entrenamiento de gatos (1000 imgs)\n os.mkdir(train_dir + '/cats')\n train_cats_dir = os.path.join(train_dir, 'cats')\n os.stat(train_cats_dir)\n\n # Directorio entrenamiento de perros (1000 imgs)\n os.mkdir(train_dir + '/dogs')\n train_dogs_dir = os.path.join(train_dir, 'dogs')\n os.stat(train_dogs_dir)\n\n # Directorio validacion de gatos (500 imgs)\n os.mkdir(validation_dir + '/cats')\n validation_cats_dir = os.path.join(validation_dir, 'cats')\n os.stat(validation_cats_dir)\n\n # Directorio validacion de perros (500 imgs)\n os.mkdir(validation_dir + '/dogs')\n validation_dogs_dir = os.path.join(validation_dir, 'dogs')\n os.stat(validation_dogs_dir)\n\n # Directorio validacion de gatos (500 imgs)\n os.mkdir(test_dir + '/cats')\n test_cats_dir = os.path.join(test_dir, 'cats')\n os.stat(test_cats_dir)\n\n # Directorio validacion de perros (500 imgs)\n os.mkdir(test_dir + '/dogs')\n test_dogs_dir = os.path.join(test_dir, 'dogs')\n os.stat(test_dogs_dir)\n \n print('Estructura de directorios completa')\n \nexcept:\n print('Estructura de directorios incompleta')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:07.009829Z","iopub.execute_input":"2023-04-15T07:21:07.010930Z","iopub.status.idle":"2023-04-15T07:21:07.022916Z","shell.execute_reply.started":"2023-04-15T07:21:07.010889Z","shell.execute_reply":"2023-04-15T07:21:07.021763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data is extracted from the respective zip found in the inputs section of the notebook, and we supress the output of the code because its very large
","metadata":{}},{"cell_type":"code","source":"@contextmanager\ndef suppress_stdout():\n with open(os.devnull, \"w\") as devnull:\n old_stdout = sys.stdout\n sys.stdout = devnull\n try: \n yield\n finally:\n sys.stdout = old_stdout\n\nwith suppress_stdout():\n ! unzip \"../input/dogs-vs-cats/test1.zip\" -d data_test\n\n ! unzip \"../input/dogs-vs-cats/train.zip\" -d data_train\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:09.818007Z","iopub.execute_input":"2023-04-15T07:21:09.818539Z","iopub.status.idle":"2023-04-15T07:21:28.395959Z","shell.execute_reply.started":"2023-04-15T07:21:09.818473Z","shell.execute_reply":"2023-04-15T07:21:28.394105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(base_dir + \"/data_train\"))\nprint(os.listdir(base_dir + \"/data_test\"))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:28.403825Z","iopub.execute_input":"2023-04-15T07:21:28.407019Z","iopub.status.idle":"2023-04-15T07:21:28.418582Z","shell.execute_reply.started":"2023-04-15T07:21:28.406957Z","shell.execute_reply":"2023-04-15T07:21:28.417377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we copy the files from the folders extracted from the zip to the previously created folders
","metadata":{}},{"cell_type":"code","source":"base_dir = '/kaggle/working/'\npath_data_train = os.path.join(base_dir, 'data_train/train/')\nfor p in os.listdir(path_data_train):\n if p.startswith('cat'):\n shutil.move(path_data_train+p, train_cats_dir)\n elif p.startswith('dog'):\n shutil.move(path_data_train+p, train_dogs_dir)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:28.420739Z","iopub.execute_input":"2023-04-15T07:21:28.421315Z","iopub.status.idle":"2023-04-15T07:21:29.587291Z","shell.execute_reply.started":"2023-04-15T07:21:28.421269Z","shell.execute_reply":"2023-04-15T07:21:29.586251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('total training cat images:', len(os.listdir(train_cats_dir)))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:29.590039Z","iopub.execute_input":"2023-04-15T07:21:29.590329Z","iopub.status.idle":"2023-04-15T07:21:29.605192Z","shell.execute_reply.started":"2023-04-15T07:21:29.590302Z","shell.execute_reply":"2023-04-15T07:21:29.604232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 3. Data Preprocessing\n\nWe will reduce the size of the database because the execution times can take a long time, but in case of obtaining more precision, the ideal would be to train the networks with all the data
","metadata":{}},{"cell_type":"code","source":"def mover(dir_from, dir_to, n):\n for i, p in enumerate(os.listdir(dir_from)):\n if i == n: break\n shutil.move(dir_from+'/'+p, dir_to)\n\ndef delete(dir, n):\n for i, p in enumerate(os.listdir(dir)):\n if i == n: break\n os.remove(dir+'/'+p)\n\ndef resize_local(dir):\n for p in os.listdir(dir):\n img = plt.imread(dir+'/'+p)\n img = cv2.resize(img, (150, 150))\n io.imsave(dir+'/'+p, img)\n \n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:29.606664Z","iopub.execute_input":"2023-04-15T07:21:29.607102Z","iopub.status.idle":"2023-04-15T07:21:29.614613Z","shell.execute_reply.started":"2023-04-15T07:21:29.607065Z","shell.execute_reply":"2023-04-15T07:21:29.613464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mover(train_cats_dir, test_cats_dir, 500)\nmover(train_cats_dir, validation_cats_dir, 500)\nmover(train_dogs_dir, test_dogs_dir, 500)\nmover(train_dogs_dir, validation_dogs_dir, 500)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:29.616362Z","iopub.execute_input":"2023-04-15T07:21:29.616804Z","iopub.status.idle":"2023-04-15T07:21:29.727176Z","shell.execute_reply.started":"2023-04-15T07:21:29.616763Z","shell.execute_reply":"2023-04-15T07:21:29.726260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delete(train_cats_dir, 10500)\ndelete(train_dogs_dir, 10500)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:29.728392Z","iopub.execute_input":"2023-04-15T07:21:29.728736Z","iopub.status.idle":"2023-04-15T07:21:30.326247Z","shell.execute_reply.started":"2023-04-15T07:21:29.728702Z","shell.execute_reply":"2023-04-15T07:21:30.325044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('total training cat images:', len(os.listdir(train_cats_dir)))\nprint('total testing cat images:', len(os.listdir(test_cats_dir)))\nprint('total validation cat images:', len(os.listdir(validation_cats_dir)))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:30.331845Z","iopub.execute_input":"2023-04-15T07:21:30.332687Z","iopub.status.idle":"2023-04-15T07:21:30.344126Z","shell.execute_reply.started":"2023-04-15T07:21:30.332650Z","shell.execute_reply":"2023-04-15T07:21:30.343089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resize_local(train_cats_dir)\nresize_local(train_dogs_dir)\nresize_local(validation_cats_dir)\nresize_local(validation_dogs_dir)\nresize_local(test_cats_dir)\nresize_local(test_dogs_dir)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:33.366333Z","iopub.execute_input":"2023-04-15T07:21:33.366807Z","iopub.status.idle":"2023-04-15T07:21:50.677070Z","shell.execute_reply.started":"2023-04-15T07:21:33.366771Z","shell.execute_reply":"2023-04-15T07:21:50.676034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A visualization of how the observations were after the preprocessing would be:
","metadata":{}},{"cell_type":"code","source":"x = os.listdir(train_cats_dir)\nplt.imshow(plt.imread(train_cats_dir + '/' + x[0])) ","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:50.678932Z","iopub.execute_input":"2023-04-15T07:21:50.679376Z","iopub.status.idle":"2023-04-15T07:21:51.018530Z","shell.execute_reply.started":"2023-04-15T07:21:50.679340Z","shell.execute_reply":"2023-04-15T07:21:51.017557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 4. Model Selection and Training","metadata":{}},{"cell_type":"markdown","source":"*Creation and Training of the Convolutional NN from scratch with keras
\n\nWe will start by creating a convolutional neural network from scratch to see what results it produces. The convolutional stage will consist of 4 main convolution layers and 4 pulling layers and the classification stage will have a layer of 562 neurons with `relu` activation function and the output layer with 1 neuron and `sigmoid` activation function, since is a binary classification problem
\n\n\n","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), \n activation='relu', \n input_shape=(150, 150, 3)))\nmodel.add(MaxPooling2D((2, 2)))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D((2, 2)))\nmodel.add(Conv2D(128, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D((2, 2)))\nmodel.add(Conv2D(128, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D((2, 2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(562, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:51.019664Z","iopub.execute_input":"2023-04-15T07:21:51.020003Z","iopub.status.idle":"2023-04-15T07:21:53.969353Z","shell.execute_reply.started":"2023-04-15T07:21:51.019968Z","shell.execute_reply":"2023-04-15T07:21:53.962700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the model, the `RSMprop` algorithm will be used with a learning rate of $10^{-4}$. The cost function will be binary crossentropy, since it is a binary classification problem.
\n","metadata":{}},{"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n optimizer= RMSprop(learning_rate=1e-4),\n metrics=['acc'])","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:53.972202Z","iopub.execute_input":"2023-04-15T07:21:53.972605Z","iopub.status.idle":"2023-04-15T07:21:54.001766Z","shell.execute_reply.started":"2023-04-15T07:21:53.972551Z","shell.execute_reply":"2023-04-15T07:21:54.000648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pixel values of each photo are rescaled to values between $[0, 1]$
","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255)\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\nprint(\"Entrenamiento\")\ntrain_generator = train_datagen.flow_from_directory(\n train_dir,\n target_size=(150, 150),\n batch_size=20,\n class_mode='binary')\n\nprint(\"\\n Validacion\")\n\nvalidation_generator = test_datagen.flow_from_directory(\n validation_dir,\n target_size=(150, 150),\n batch_size=20,\n class_mode='binary')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:54.003355Z","iopub.execute_input":"2023-04-15T07:21:54.004014Z","iopub.status.idle":"2023-04-15T07:21:54.222812Z","shell.execute_reply.started":"2023-04-15T07:21:54.003976Z","shell.execute_reply":"2023-04-15T07:21:54.221771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,steps_per_epoch=100,\n epochs=30,\n validation_data=validation_generator, \n validation_steps=50, verbose=1,\n workers = 12)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:21:54.226242Z","iopub.execute_input":"2023-04-15T07:21:54.226551Z","iopub.status.idle":"2023-04-15T07:26:32.925758Z","shell.execute_reply.started":"2023-04-15T07:21:54.226523Z","shell.execute_reply":"2023-04-15T07:26:32.924749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.figure(figsize=(8,10))\n\nplt.subplot(211)\nplt.plot(epochs, acc, 'bo', label='Entrenamiento acc')\nplt.plot(epochs, val_acc, 'b', label='Validacion acc')\nplt.title('Exactitud (accuracy) de entrenamiento y validacion')\nplt.legend()\n\nplt.subplot(212)\nplt.plot(epochs, loss, 'bo', label='Entrenamiento costo')\nplt.plot(epochs, val_loss, 'b', label='Validacion cost')\nplt.title('Costo (loss) de entrenamiento y validacion')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:26:32.927531Z","iopub.execute_input":"2023-04-15T07:26:32.928017Z","iopub.status.idle":"2023-04-15T07:26:33.276624Z","shell.execute_reply.started":"2023-04-15T07:26:32.927978Z","shell.execute_reply":"2023-04-15T07:26:33.275549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An overfitting can be observed in the graphs, it was something to be expected, since data augmentation was not even applied to the observations, only the pixel values were rescaled.
\n\n","metadata":{}},{"cell_type":"markdown","source":"*Offline Trait Extractor
\n\nTo test another strategy, we will use knowledge transfer with the `VGG16` network of `Imagnet`. As a first strategy, the Pretrained Network will be applied as an extractor of off-line features, that is, all the convolutional layers of the pretrained network will be applied to the data to then create a new dataset and manually define the classification stage, as a multilayer perceptron.
\n","metadata":{}},{"cell_type":"code","source":"conv_base = VGG16(weights='imagenet',\n include_top=False,\n input_shape=(150, 150, 3))\n\ndatagen = ImageDataGenerator(preprocessing_function=preprocess_input)\n\nbatch_size = 20\n\ndef extract_features(directory, sample_count):\n features = np.zeros(shape=(sample_count, 4, 4, 512))\n labels = np.zeros(shape=(sample_count))\n generator = datagen.flow_from_directory(\n directory,\n target_size=(150, 150),\n batch_size=batch_size,\n class_mode='binary')\n i = 0\n for inputs_batch, labels_batch in generator:\n features_batch = conv_base.predict(inputs_batch)\n features[i * batch_size : (i + 1) * batch_size] = features_batch\n labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n i += 1\n if i * batch_size >= sample_count:\n break\n return features, labels\n\ntrain_features, train_labels = extract_features(train_dir, 2000)\nvalidation_features, validation_labels = extract_features(validation_dir, 1000)\ntest_features, test_labels = extract_features(test_dir, 1000)\n\ntrain_features = np.reshape(train_features, (2000, 4 * 4 * 512))\nvalidation_features = np.reshape(validation_features, (1000, 4 * 4 * 512))\ntest_features = np.reshape(test_features, (1000, 4 * 4 * 512))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:26:33.280081Z","iopub.execute_input":"2023-04-15T07:26:33.281197Z","iopub.status.idle":"2023-04-15T07:27:00.203987Z","shell.execute_reply.started":"2023-04-15T07:26:33.281156Z","shell.execute_reply":"2023-04-15T07:27:00.202911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(256, activation='relu', input_dim=4 * 4 * 512))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(optimizer=RMSprop(learning_rate=2e-5),\n loss='binary_crossentropy',\n metrics=['acc'])\n\nhistory = model.fit(train_features, train_labels,\n epochs = 30,\n batch_size = 20,\n validation_data = (validation_features, validation_labels),\n verbose = 2)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:27:00.205320Z","iopub.execute_input":"2023-04-15T07:27:00.205942Z","iopub.status.idle":"2023-04-15T07:27:11.575941Z","shell.execute_reply.started":"2023-04-15T07:27:00.205904Z","shell.execute_reply":"2023-04-15T07:27:11.574926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Entrenamiento acc')\nplt.plot(epochs, val_acc, 'b', label='Validación acc')\nplt.title('Accuracy - exactitud de entrenamiento y validación')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Entrenamiento loss')\nplt.plot(epochs, val_loss, 'b', label='Validación loss')\nplt.title('Loss - función objetivo en entrenamiento y prueba')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:27:11.579513Z","iopub.execute_input":"2023-04-15T07:27:11.579851Z","iopub.status.idle":"2023-04-15T07:27:12.022536Z","shell.execute_reply.started":"2023-04-15T07:27:11.579823Z","shell.execute_reply":"2023-04-15T07:27:12.021522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Frozen Convolutional Training
\n\nNow to see if the precision of the model increases further, we will take the convolutional stage of the neural network and leave it fixed, but we will train the entire model, passing the data in each epoch through the convolutional stage of the pretrained network, with frozen parameters.
","metadata":{}},{"cell_type":"code","source":"conv_base.trainable = False\n\nmodel = Sequential()\nmodel.add(conv_base) # modelo base agradado como una capa!\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(2, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:27:12.024104Z","iopub.execute_input":"2023-04-15T07:27:12.026822Z","iopub.status.idle":"2023-04-15T07:27:12.125419Z","shell.execute_reply.started":"2023-04-15T07:27:12.026791Z","shell.execute_reply":"2023-04-15T07:27:12.124405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,\n rotation_range = 40,\n width_shift_range = 0.2,\n height_shift_range= 0.2,\n shear_range = 0.2,\n zoom_range = 0.2,\n horizontal_flip = True,\n fill_mode = 'constant',\n cval = 0)\n\n# La validación no se aumenta!\ntest_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)\n\ntrain_generator = train_datagen.flow_from_directory(\n train_dir, # directorio con datos de entrenamiento\n target_size= (150, 150), # tamaño de la imágenes \n batch_size = 20, \n shuffle = True,\n class_mode = 'categorical') # para clasificación binaria\n\nvalidation_generator = test_datagen.flow_from_directory(\n validation_dir,\n target_size=(150, 150),\n batch_size=20,\n shuffle = False,\n class_mode='categorical')\n\nmodel.compile(loss='categorical_crossentropy',\n optimizer= Nadam(learning_rate=2e-5), #optimizers.RMSprop(lr=2e-5),\n metrics=['acc'])\n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:27:12.127138Z","iopub.execute_input":"2023-04-15T07:27:12.127496Z","iopub.status.idle":"2023-04-15T07:27:12.351389Z","shell.execute_reply.started":"2023-04-15T07:27:12.127459Z","shell.execute_reply":"2023-04-15T07:27:12.350371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n steps_per_epoch = 100, \n epochs = 30,\n validation_data = validation_generator,\n validation_steps= 50,\n verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:27:12.352793Z","iopub.execute_input":"2023-04-15T07:27:12.353228Z","iopub.status.idle":"2023-04-15T07:33:45.192849Z","shell.execute_reply.started":"2023-04-15T07:27:12.353191Z","shell.execute_reply":"2023-04-15T07:33:45.191799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Even more precision is achieved than the previous model
","metadata":{}},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Entrenamiento acc')\nplt.plot(epochs, val_acc, 'b', label='Validación acc')\nplt.title('Accuracy - exactitud de entrenamiento y validación')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Entrenamiento loss')\nplt.plot(epochs, val_loss, 'b', label='Validación loss')\nplt.title('Loss - función objetivo en entrenamiento y prueba')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:33:45.194533Z","iopub.execute_input":"2023-04-15T07:33:45.195145Z","iopub.status.idle":"2023-04-15T07:33:45.615524Z","shell.execute_reply.started":"2023-04-15T07:33:45.195107Z","shell.execute_reply":"2023-04-15T07:33:45.614553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Fine tuning
\n\nNow we will perform a fine adjustment to the last 3 convolution layers of the `VGG16` network:
\n \n1. `block5_conv2`\n2. `block5_conv3`\n3. `block5_pool`\n\nFinally we will train the model, with the same classification stage used in all the other two models.
\n","metadata":{}},{"cell_type":"code","source":"conv_base.trainable = True\n\nfor layer in conv_base.layers:\n if layer.name in ['block5_conv1','block5_conv2','block5_conv3']:\n layer.trainable = True\n else:\n layer.trainable = False \n\nmodel.compile(loss='categorical_crossentropy',\n optimizer=RMSprop(learning_rate=1e-5),\n metrics=['acc'])","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:33:45.616995Z","iopub.execute_input":"2023-04-15T07:33:45.617446Z","iopub.status.idle":"2023-04-15T07:33:45.632004Z","shell.execute_reply.started":"2023-04-15T07:33:45.617409Z","shell.execute_reply":"2023-04-15T07:33:45.630959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n steps_per_epoch=100,\n epochs=100,\n validation_data=validation_generator,\n validation_steps=50)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:33:45.633742Z","iopub.execute_input":"2023-04-15T07:33:45.634257Z","iopub.status.idle":"2023-04-15T07:54:56.997463Z","shell.execute_reply.started":"2023-04-15T07:33:45.634219Z","shell.execute_reply":"2023-04-15T07:54:56.996448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Entrenamiento acc')\nplt.plot(epochs, val_acc, 'b', label='Validación acc')\nplt.title('Accuracy - exactitud de entrenamiento y validación')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Entrenamiento loss')\nplt.plot(epochs, val_loss, 'b', label='Validación loss')\nplt.title('Loss - función objetivo en entrenamiento y prueba')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:54:57.002170Z","iopub.execute_input":"2023-04-15T07:54:57.002459Z","iopub.status.idle":"2023-04-15T07:54:57.427559Z","shell.execute_reply.started":"2023-04-15T07:54:57.002431Z","shell.execute_reply":"2023-04-15T07:54:57.426523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the curves are very noisy, it is possible to filter them to better appreciate the trend by means of a low pass filter, one could try
\n\n$$\\hat s_i = \\sum_{k = 0}^{n}w_k s_{i - k}$$\n\nwhere $w$ are the weights with $\\sum_{k}w_k = 1$ but we would need to use a fairly wide filter ($n$ large) to properly remove noise, instead let's try the recursive version
\n\n$$\\hat s = \\alpha\\hat s_{i - 1} + (1 - \\alpha)\\hat s_i$$\n\nthat is, a convex linear combination of the points
","metadata":{}},{"cell_type":"code","source":"def smooth_curve(points, factor=0.8):\n smoothed_points = []\n for point in points:\n if smoothed_points:\n previous = smoothed_points[-1]\n smoothed_points.append(previous * factor + point * (1 - factor))\n else:\n smoothed_points.append(point)\n return smoothed_points\n\nacc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\nalpha = 0.9\n\nplt.plot(epochs, smooth_curve(acc, alpha), 'bo', label='Entrenamiento acc')\nplt.plot(epochs, smooth_curve(val_acc, alpha), 'b', label='Validación acc')\nplt.title('Accuracy - exactitud de entrenamiento y validación')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, smooth_curve(loss, alpha), 'bo', label='Entrenamiento loss')\nplt.plot(epochs, smooth_curve(val_loss,alpha), 'b', label='Validación loss')\nplt.title('Loss - función objetivo en entrenamiento y prueba')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:54:57.429177Z","iopub.execute_input":"2023-04-15T07:54:57.429523Z","iopub.status.idle":"2023-04-15T07:54:57.855507Z","shell.execute_reply.started":"2023-04-15T07:54:57.429487Z","shell.execute_reply":"2023-04-15T07:54:57.854512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In these curves you can better appreciate the trend, we can see that there was an improvement of around 1%. It is necessary to appreciate that the accuracy is improving even when the loss is not. This is because the loss is evaluated by a process of summation of errors in the output vector, something like:
\n\n$$\\sum_{i \\in epocs}||y_i - \\hat y_i||$$\n\ninstead the accuracy is calculated by a sum of classification errors:
\n\n$$\\sum_{i \\in epocs}1 - \\delta(\\arg \\max_i y_i - \\arg \\max_i \\hat y_i )$$\n\n","metadata":{}},{"cell_type":"markdown","source":"\n# 5. GrandCAM visualizations","metadata":{}},{"cell_type":"markdown","source":"GranCAM (Gradient-weighted Class Activation Mapping) visualizations are a convolutional neural network (ConvNN) visualization technique that allows us to understand which parts of an image are relevant for network decision making.
\n\nThis technique uses gradient backpropagation to calculate the importance of each image pixel in the final classification performed by the neural network. These importances are then applied as weights on a weighted sum of the activations of the last convolutional layer of the network, resulting in a heat map image showing the areas of the image that most influence the classification of the network. .
\n\nIn this way, GranCAM visualizations allow a more intuitive interpretation of the output of a convolutional neural network, which can be useful for model analysis and debugging, as well as for understanding network decision making in case artificial intelligence applications in the real world.
\n\nFirstly we will define the function that will calculate the probability map, and the saliency, which is no more than the average response of the last convolutional layer
\n\n","metadata":{}},{"cell_type":"code","source":"def make_gradcam_heatmap(img_array, model, last_conv_layer_name, classifier_layer_names):\n \n last_conv_layer = model.get_layer(last_conv_layer_name)\n conv_model = keras.Model(last_conv_layer.inputs, last_conv_layer.output)\n \n classifier_input = keras.Input(shape=last_conv_layer.output.shape[1:])\n x = classifier_input\n for layer_name in classifier_layer_names:\n x = model.get_layer(layer_name)(x)\n classifier_model = keras.Model(classifier_input, x)\n \n with tf.GradientTape() as tape:\n last_conv_layer_output = conv_model(img_array)\n tape.watch(last_conv_layer_output)\n \n preds = classifier_model(last_conv_layer_output)\n top_pred_index = tf.argmax(preds[0])\n top_class_channel = preds[:, top_pred_index]\n \n grads = tape.gradient(top_class_channel, last_conv_layer_output)\n pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n last_conv_layer_output = last_conv_layer_output.numpy()[0]\n \n saliency = np.mean(last_conv_layer_output, axis=-1)\n saliency = np.maximum(saliency, 0) / np.max(saliency)\n \n pooled_grads = pooled_grads.numpy()\n for i in range(pooled_grads.shape[-1]):\n last_conv_layer_output[:, :, i] *= pooled_grads[i]\n \n grad_cam = np.mean(last_conv_layer_output, axis=-1)\n grad_cam = np.maximum(grad_cam, 0)/ np.max(grad_cam)\n \n return grad_cam, saliency","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:17:01.466556Z","iopub.execute_input":"2023-04-15T08:17:01.467125Z","iopub.status.idle":"2023-04-15T08:17:01.483544Z","shell.execute_reply.started":"2023-04-15T08:17:01.467092Z","shell.execute_reply":"2023-04-15T08:17:01.482356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We load any photo from the test set and convert it to a numpy array
","metadata":{}},{"cell_type":"code","source":"def get_img_array(img_path):\n \n img = Image.open(img_path)\n img = img.resize(size=(150, 150))\n \n print(f'format: {img.format}, shape: {img.size}, mode: {img.mode}')\n img_array = np.array(img).astype('float32')[:,:,:3] # tiramos el canal alpha\n print(img_array.shape)\n img_array = np.expand_dims(img_array, axis=0)\n print(img_array.shape)\n return img, img_array","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_conv_layer_name = model.layers[0].name\nclassifier_layer_names = []\nfor i in range(1, len(model.layers)):\n classifier_layer_names.append(model.layers[i].name)\n\n# This is because there are a few pictures that have problem with my function,\n# and I have to run all notebook, so I simply search the first that is compatible\ndata_dir = '/kaggle/working/data_test/test1'\ni = 0\nwhile True:\n img_path = os.path.join(data_dir, os.listdir(data_dir)[i])\n img,img_array = get_img_array(img_path = img_path)\n grad_cam, saliency = make_gradcam_heatmap(img_array, \n model, \n last_conv_layer_name, \n classifier_layer_names)\n if not np.all(np.isnan(grad_cam)): break\n i += 1\n \nplt.imshow(img)\nplt.show()\n\nprint(\" GradCAM: \\n\")\nprint(grad_cam) ","metadata":{"execution":{"iopub.status.busy":"2023-04-15T09:15:34.916894Z","iopub.execute_input":"2023-04-15T09:15:34.917613Z","iopub.status.idle":"2023-04-15T09:15:35.261317Z","shell.execute_reply.started":"2023-04-15T09:15:34.917551Z","shell.execute_reply":"2023-04-15T09:15:35.260394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_array = preprocess_input(img_array)\npreds = model.predict(img_array, verbose=1)\npreds","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:33:15.952200Z","iopub.execute_input":"2023-04-15T08:33:15.952883Z","iopub.status.idle":"2023-04-15T08:33:16.020558Z","shell.execute_reply.started":"2023-04-15T08:33:15.952846Z","shell.execute_reply":"2023-04-15T08:33:16.019505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_hotmap(img, heatmap, title='Heatmap', alpha=0.6, cmap='jet', axisOnOff='off'):\n \n resized_heatmap=resize(heatmap, img.size)\n \n fig, ax = plt.subplots()\n ax.imshow(img)\n ax.imshow(resized_heatmap, alpha=alpha, cmap=cmap)\n plt.axis(axisOnOff)\n plt.title(title)\n plt.show()\n \nplt.subplot(121)\nplt.imshow(grad_cam, 'jet')\nplt.title('GradCam')\nplt.subplot(122)\nplt.imshow(saliency, 'jet')\nplt.title('Saliencia')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:33:18.544387Z","iopub.execute_input":"2023-04-15T08:33:18.545095Z","iopub.status.idle":"2023-04-15T08:33:18.818609Z","shell.execute_reply.started":"2023-04-15T08:33:18.545057Z","shell.execute_reply":"2023-04-15T08:33:18.817521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_hotmap(img=img, heatmap=grad_cam, title=f'Grad Cam: {model.name}')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:33:20.721529Z","iopub.execute_input":"2023-04-15T08:33:20.722244Z","iopub.status.idle":"2023-04-15T08:33:20.972406Z","shell.execute_reply.started":"2023-04-15T08:33:20.722207Z","shell.execute_reply":"2023-04-15T08:33:20.971495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_hotmap(img=img, heatmap=saliency, title=f'Saliencia: {model.name}')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T08:30:38.345435Z","iopub.execute_input":"2023-04-15T08:30:38.345872Z","iopub.status.idle":"2023-04-15T08:30:38.589525Z","shell.execute_reply.started":"2023-04-15T08:30:38.345833Z","shell.execute_reply":"2023-04-15T08:30:38.588506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note how GradCam effectively displays the activation map for the selected class, in our example we automatically select the one with the highest probability. While the saliency shows the heat maps for the most probable classes.
\n\n\n","metadata":{}},{"cell_type":"markdown","source":"\n# 6. Evaluating Models\n\nFor the model evaluation process, the following metrics will be calculated based on the confusion matrix
\n\n1. `Accuracy` \n2. `Sensitivity` (o recall)\n3. `Specificity`\n4. `Presicion`\n5. `Confussion Matrix`\n6. `AUC-ROC` (area under the curve ROC)","metadata":{}},{"cell_type":"code","source":"test_generator = test_datagen.flow_from_directory(\n test_dir,\n target_size=(150, 150),\n batch_size=20,\n shuffle = False,\n class_mode='categorical')\n\ntest_loss, test_acc = model.evaluate(test_generator, steps=50)\nprint('test acc:', test_acc)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:59:22.232847Z","iopub.execute_input":"2023-04-15T07:59:22.233243Z","iopub.status.idle":"2023-04-15T07:59:24.959789Z","shell.execute_reply.started":"2023-04-15T07:59:22.233211Z","shell.execute_reply":"2023-04-15T07:59:24.958496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(test_generator, verbose=1)\ny_true = test_generator.classes\n\nconf_mat = confusion_matrix(y_true, np.argmax(y_pred, axis=1))\nprint('Confusion matrix:')\nprint(conf_mat)\n\naccuracy = (conf_mat[0,0]+conf_mat[1,1])/sum(sum(conf_mat))\nprint('Accuracy:', accuracy)\n\nspecificity = conf_mat[0,0]/(conf_mat[0,0]+conf_mat[0,1])\nprint('Specificity:', specificity)\n\nrecall = conf_mat[1,1]/(conf_mat[1,0]+conf_mat[1,1])\nprint('Sensitivity:', recall)\n\nprecision = conf_mat[1,1]/(conf_mat[0,1]+conf_mat[1,1])\nprint('Precision:', precision)\n\nAUC = roc_auc_score(y_true, y_pred[:,1])\nprint('AUC-ROC:', AUC)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T07:59:26.239460Z","iopub.execute_input":"2023-04-15T07:59:26.240162Z","iopub.status.idle":"2023-04-15T07:59:27.839999Z","shell.execute_reply.started":"2023-04-15T07:59:26.240124Z","shell.execute_reply":"2023-04-15T07:59:27.838853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## I hope you liked it, give an upvote in that case :)","metadata":{}}]}