{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Building Datasets**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n#codes are taken from different open source sites\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls \"../input/train/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport os\nimport shutil\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = glob.glob('../input/train/*')\n\ncat_files = [fn for fn in files if 'cat' in fn]\ndog_files = [fn for fn in files if 'dog' in fn]\nlen(cat_files), len(dog_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_train = np.random.choice(cat_files, size=1500, replace=False)\ndog_train = np.random.choice(dog_files, size=1500, replace=False)\ncat_files = list(set(cat_files) - set(cat_train))\ndog_files = list(set(dog_files) - set(dog_train))\n\ncat_val = np.random.choice(cat_files, size=500, replace=False)\ndog_val = np.random.choice(dog_files, size=500, replace=False)\ncat_files = list(set(cat_files) - set(cat_val))\ndog_files = list(set(dog_files) - set(dog_val))\n\ncat_test = np.random.choice(cat_files, size=500, replace=False)\ndog_test = np.random.choice(dog_files, size=500, replace=False)\n\nprint('Cat datasets:', cat_train.shape, cat_val.shape, cat_test.shape)\nprint('Dog datasets:', dog_train.shape, dog_val.shape, dog_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = 'training_data'\nval_dir = 'validation_data'\ntest_dir = 'test_data'\n\ntrain_files = np.concatenate([cat_train, dog_train])\nvalidate_files = np.concatenate([cat_val, dog_val])\ntest_files = np.concatenate([cat_test, dog_test])\n\nos.mkdir(train_dir) if not os.path.isdir(train_dir) else None\nos.mkdir(val_dir) if not os.path.isdir(val_dir) else None\nos.mkdir(test_dir) if not os.path.isdir(test_dir) else None\n\nfor fn in train_files:\n    shutil.copy(fn, train_dir)\n\nfor fn in validate_files:\n    shutil.copy(fn, val_dir)\n    \nfor fn in test_files:\n    shutil.copy(fn, test_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preparing Datasets**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_DIM = (150, 150)\n\ntrain_files = glob.glob('training_data/*')\ntrain_imgs = [img_to_array(load_img(img, target_size=IMG_DIM)) for img in train_files]\ntrain_imgs = np.array(train_imgs)\ntrain_labels = [fn.split('/')[1].split('.')[0].strip() for fn in train_files]\n\nvalidation_files = glob.glob('validation_data/*')\nvalidation_imgs = [img_to_array(load_img(img, target_size=IMG_DIM)) for img in validation_files]\nvalidation_imgs = np.array(validation_imgs)\nvalidation_labels = [fn.split('/')[1].split('.')[0].strip() for fn in validation_files]\n\nprint('Train dataset shape:', train_imgs.shape, \n      '\\tValidation dataset shape:', validation_imgs.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs_scaled = train_imgs.astype('float32')\nvalidation_imgs_scaled  = validation_imgs.astype('float32')\ntrain_imgs_scaled /= 255\nvalidation_imgs_scaled /= 255\n\nprint(train_imgs[0].shape)\narray_to_img(train_imgs[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 2\nepochs = 30\ninput_shape = (150, 150, 3)\n\n# encode text category labels\nfrom sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\nle.fit(train_labels)\ntrain_labels_enc = le.transform(train_labels)\nvalidation_labels_enc = le.transform(validation_labels)\n\nprint(train_labels[1495:1505], train_labels_enc[1495:1505])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Simple CNN Model from Scratch**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom keras.models import Sequential\nfrom keras import optimizers\n\nmodel = Sequential()\n\nmodel.add(Conv2D(16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(x=train_imgs_scaled, y=train_labels_enc,\n                    validation_data=(validation_imgs_scaled, validation_labels_enc),\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Basic CNN Performance', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(1,31))\nax1.plot(epoch_list, history.history['acc'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_acc'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 31, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 31, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CNN Model with Regularization**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))\n\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n              \n              \nhistory = model.fit(x=train_imgs_scaled, y=train_labels_enc,\n                    validation_data=(validation_imgs_scaled, validation_labels_enc),\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    verbose=1)         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cats_dogs_basic_cnn.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CNN Model with Image Augmentation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255, zoom_range=0.3, rotation_range=50,\n                                   width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, \n                                   horizontal_flip=True, fill_mode='nearest')\n\nval_datagen = ImageDataGenerator(rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_id = 2595\ncat_generator = train_datagen.flow(train_imgs[img_id:img_id+1], train_labels[img_id:img_id+1],\n                                   batch_size=1)\ncat = [next(cat_generator) for i in range(0,5)]\nfig, ax = plt.subplots(1,5, figsize=(16, 6))\nprint('Labels:', [item[1][0] for item in cat])\nl = [ax[i].imshow(cat[i][0][0]) for i in range(0,5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_id = 1991\ndog_generator = train_datagen.flow(train_imgs[img_id:img_id+1], train_labels[img_id:img_id+1],\n                                   batch_size=1)\ndog = [next(dog_generator) for i in range(0,5)]\nfig, ax = plt.subplots(1,5, figsize=(15, 6))\nprint('Labels:', [item[1][0] for item in dog])\nl = [ax[i].imshow(dog[i][0][0]) for i in range(0,5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = train_datagen.flow(train_imgs, train_labels_enc, batch_size=30)\nval_generator = val_datagen.flow(validation_imgs, validation_labels_enc, batch_size=20)\ninput_shape = (150, 150, 3)\n\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom keras.models import Sequential\nfrom keras import optimizers\n\nmodel = Sequential()\n\nmodel.add(Conv2D(16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=1e-4),\n              metrics=['accuracy'])\n              \nhistory = model.fit_generator(train_generator, steps_per_epoch=100, epochs=100,\n                              validation_data=val_generator, validation_steps=50, \n                              verbose=1)              ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cats_dogs_cnn_img_aug.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Leveraging Transfer Learning with Pre-trained CNN Models**"},{"metadata":{},"cell_type":"markdown","source":"***Pre-trained CNN model as a Feature Extractor***"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import vgg16\nfrom keras.models import Model\nimport keras\n\nvgg = vgg16.VGG16(include_top=False, weights='imagenet', \n                                     input_shape=input_shape)\n\noutput = vgg.layers[-1].output\noutput = keras.layers.Flatten()(output)\nvgg_model = Model(vgg.input, output)\n\nvgg_model.trainable = False\nfor layer in vgg_model.layers:\n    layer.trainable = False\n    \nimport pandas as pd\npd.set_option('max_colwidth', -1)\nlayers = [(layer, layer.name, layer.trainable) for layer in vgg_model.layers]\npd.DataFrame(layers, columns=['Layer Type', 'Layer Name', 'Layer Trainable'])    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bottleneck_feature_example = vgg.predict(train_imgs_scaled[0:1])\nprint(bottleneck_feature_example.shape)\nplt.imshow(bottleneck_feature_example[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bottleneck_features(model, input_imgs):\n    features = model.predict(input_imgs, verbose=0)\n    return features\n    \ntrain_features_vgg = get_bottleneck_features(vgg_model, train_imgs_scaled)\nvalidation_features_vgg = get_bottleneck_features(vgg_model, validation_imgs_scaled)\n\nprint('Train Bottleneck Features:', train_features_vgg.shape, \n      '\\tValidation Bottleneck Features:', validation_features_vgg.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, InputLayer\nfrom keras.models import Sequential\nfrom keras import optimizers\n\ninput_shape = vgg_model.output_shape[1]\n\nmodel = Sequential()\nmodel.add(InputLayer(input_shape=(input_shape,)))\nmodel.add(Dense(512, activation='relu', input_dim=input_shape))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=1e-4),\n              metrics=['accuracy'])\n\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(x=train_features_vgg, y=train_labels_enc,\n                    validation_data=(validation_features_vgg, validation_labels_enc),\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cats_dogs_tlearn_basic_cnn.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***Pre-trained CNN model as a Feature Extractor with Image Augmentation***"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255, zoom_range=0.3, rotation_range=50,\n                                   width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, \n                                   horizontal_flip=True, fill_mode='nearest')\n\nval_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_generator = train_datagen.flow(train_imgs, train_labels_enc, batch_size=30)\nval_generator = val_datagen.flow(validation_imgs, validation_labels_enc, batch_size=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, InputLayer\nfrom keras.models import Sequential\nfrom keras import optimizers\n\nmodel = Sequential()\nmodel.add(vgg_model)\nmodel.add(Dense(512, activation='relu', input_dim=input_shape))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=2e-5),\n              metrics=['accuracy'])\n              \nhistory = model.fit_generator(train_generator, steps_per_epoch=100, epochs=100,\n                              validation_data=val_generator, validation_steps=50, \n                              verbose=1)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cats_dogs_tlearn_img_aug_cnn.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***Pre-trained CNN model with Fine-tuning and Image Augmentation***"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_model.trainable = True\n\nset_trainable = False\nfor layer in vgg_model.layers:\n    if layer.name in ['block5_conv1', 'block4_conv1']:\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False\n        \nlayers = [(layer, layer.name, layer.trainable) for layer in vgg_model.layers]\npd.DataFrame(layers, columns=['Layer Type', 'Layer Name', 'Layer Trainable'])    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255, zoom_range=0.3, rotation_range=50,\n                                   width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, \n                                   horizontal_flip=True, fill_mode='nearest')\nval_datagen = ImageDataGenerator(rescale=1./255)\ntrain_generator = train_datagen.flow(train_imgs, train_labels_enc, batch_size=30)\nval_generator = val_datagen.flow(validation_imgs, validation_labels_enc, batch_size=20)\n\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, InputLayer\nfrom keras.models import Sequential\nfrom keras import optimizers\n\nmodel = Sequential()\nmodel.add(vgg_model)\nmodel.add(Dense(512, activation='relu', input_dim=input_shape))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=1e-5),\n              metrics=['accuracy'])\n              \nhistory = model.fit_generator(train_generator, steps_per_epoch=100, epochs=100,\n                              validation_data=val_generator, validation_steps=50, \n                              verbose=1)         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cats_dogs_tlearn_finetune_img_aug_cnn.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.base import clone\nfrom sklearn.preprocessing import label_binarize\nfrom scipy import interp\nfrom sklearn.metrics import roc_curve, auc \n\n\ndef get_metrics(true_labels, predicted_labels):\n    \n    print('Accuracy:', np.round(\n                        metrics.accuracy_score(true_labels, \n                                               predicted_labels),\n                        4))\n    print('Precision:', np.round(\n                        metrics.precision_score(true_labels, \n                                               predicted_labels,\n                                               average='weighted'),\n                        4))\n    print('Recall:', np.round(\n                        metrics.recall_score(true_labels, \n                                               predicted_labels,\n                                               average='weighted'),\n                        4))\n    print('F1 Score:', np.round(\n                        metrics.f1_score(true_labels, \n                                               predicted_labels,\n                                               average='weighted'),\n                        4))\n                        \n\ndef train_predict_model(classifier, \n                        train_features, train_labels, \n                        test_features, test_labels):\n    # build model    \n    classifier.fit(train_features, train_labels)\n    # predict using model\n    predictions = classifier.predict(test_features) \n    return predictions    \n\n\ndef display_confusion_matrix(true_labels, predicted_labels, classes=[1,0]):\n    \n    total_classes = len(classes)\n    level_labels = [total_classes*[0], list(range(total_classes))]\n\n    cm = metrics.confusion_matrix(y_true=true_labels, y_pred=predicted_labels, \n                                  labels=classes)\n    cm_frame = pd.DataFrame(data=cm, \n                            columns=pd.MultiIndex(levels=[['Predicted:'], classes], \n                                                  labels=level_labels), \n                            index=pd.MultiIndex(levels=[['Actual:'], classes], \n                                                labels=level_labels)) \n    print(cm_frame) \n    \ndef display_classification_report(true_labels, predicted_labels, classes=[1,0]):\n\n    report = metrics.classification_report(y_true=true_labels, \n                                           y_pred=predicted_labels, \n                                           labels=classes) \n    print(report)\n    \n    \n    \ndef display_model_performance_metrics(true_labels, predicted_labels, classes=[1,0]):\n    print('Model Performance metrics:')\n    print('-'*30)\n    get_metrics(true_labels=true_labels, predicted_labels=predicted_labels)\n    print('\\nModel Classification report:')\n    print('-'*30)\n    display_classification_report(true_labels=true_labels, predicted_labels=predicted_labels, \n                                  classes=classes)\n    print('\\nPrediction Confusion Matrix:')\n    print('-'*30)\n    display_confusion_matrix(true_labels=true_labels, predicted_labels=predicted_labels, \n                             classes=classes)\n\n\ndef plot_model_decision_surface(clf, train_features, train_labels,\n                                plot_step=0.02, cmap=plt.cm.RdYlBu,\n                                markers=None, alphas=None, colors=None):\n    \n    if train_features.shape[1] != 2:\n        raise ValueError(\"X_train should have exactly 2 columnns!\")\n    \n    x_min, x_max = train_features[:, 0].min() - plot_step, train_features[:, 0].max() + plot_step\n    y_min, y_max = train_features[:, 1].min() - plot_step, train_features[:, 1].max() + plot_step\n    xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),\n                         np.arange(y_min, y_max, plot_step))\n\n    clf_est = clone(clf)\n    clf_est.fit(train_features,train_labels)\n    if hasattr(clf_est, 'predict_proba'):\n        Z = clf_est.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:,1]\n    else:\n        Z = clf_est.predict(np.c_[xx.ravel(), yy.ravel()])    \n    Z = Z.reshape(xx.shape)\n    cs = plt.contourf(xx, yy, Z, cmap=cmap)\n    \n    le = LabelEncoder()\n    y_enc = le.fit_transform(train_labels)\n    n_classes = len(le.classes_)\n    plot_colors = ''.join(colors) if colors else [None] * n_classes\n    label_names = le.classes_\n    markers = markers if markers else [None] * n_classes\n    alphas = alphas if alphas else [None] * n_classes\n    for i, color in zip(range(n_classes), plot_colors):\n        idx = np.where(y_enc == i)\n        plt.scatter(train_features[idx, 0], train_features[idx, 1], c=color,\n                    label=label_names[i], cmap=cmap, edgecolors='black', \n                    marker=markers[i], alpha=alphas[i])\n    plt.legend()\n    plt.show()\n\n\ndef plot_model_roc_curve(clf, features, true_labels, label_encoder=None, class_names=None):\n    \n    ## Compute ROC curve and ROC area for each class\n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    if hasattr(clf, 'classes_'):\n        class_labels = clf.classes_\n    elif label_encoder:\n        class_labels = label_encoder.classes_\n    elif class_names:\n        class_labels = class_names\n    else:\n        raise ValueError('Unable to derive prediction classes, please specify class_names!')\n    n_classes = len(class_labels)\n    y_test = label_binarize(true_labels, classes=class_labels)\n    if n_classes == 2:\n        if hasattr(clf, 'predict_proba'):\n            prob = clf.predict_proba(features)\n            y_score = prob[:, prob.shape[1]-1] \n        elif hasattr(clf, 'decision_function'):\n            prob = clf.decision_function(features)\n            y_score = prob[:, prob.shape[1]-1]\n        else:\n            raise AttributeError(\"Estimator doesn't have a probability or confidence scoring system!\")\n        \n        fpr, tpr, _ = roc_curve(y_test, y_score)      \n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label='ROC curve (area = {0:0.2f})'\n                                 ''.format(roc_auc),\n                 linewidth=2.5)\n        \n    elif n_classes > 2:\n        if hasattr(clf, 'predict_proba'):\n            y_score = clf.predict_proba(features)\n        elif hasattr(clf, 'decision_function'):\n            y_score = clf.decision_function(features)\n        else:\n            raise AttributeError(\"Estimator doesn't have a probability or confidence scoring system!\")\n\n        for i in range(n_classes):\n            fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n            roc_auc[i] = auc(fpr[i], tpr[i])\n\n        ## Compute micro-average ROC curve and ROC area\n        fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n        roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n\n        ## Compute macro-average ROC curve and ROC area\n        # First aggregate all false positive rates\n        all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n        # Then interpolate all ROC curves at this points\n        mean_tpr = np.zeros_like(all_fpr)\n        for i in range(n_classes):\n            mean_tpr += interp(all_fpr, fpr[i], tpr[i])\n        # Finally average it and compute AUC\n        mean_tpr /= n_classes\n        fpr[\"macro\"] = all_fpr\n        tpr[\"macro\"] = mean_tpr\n        roc_auc[\"macro\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n\n        ## Plot ROC curves\n        plt.figure(figsize=(6, 4))\n        plt.plot(fpr[\"micro\"], tpr[\"micro\"],\n                 label='micro-average ROC curve (area = {0:0.2f})'\n                       ''.format(roc_auc[\"micro\"]), linewidth=3)\n\n        plt.plot(fpr[\"macro\"], tpr[\"macro\"],\n                 label='macro-average ROC curve (area = {0:0.2f})'\n                       ''.format(roc_auc[\"macro\"]), linewidth=3)\n\n        for i, label in enumerate(class_labels):\n            plt.plot(fpr[i], tpr[i], label='ROC curve of class {0} (area = {1:0.2f})'\n                                           ''.format(label, roc_auc[i]), \n                     linewidth=2, linestyle=':')\n    else:\n        raise ValueError('Number of classes should be atleast 2 or more')\n        \n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic (ROC) Curve')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Evaluating our Deep Learning Models on Test Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load dependencies\nimport glob\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import load_img, img_to_array, array_to_img\nfrom keras.models import load_model\n#import model_evaluation_utils as meu\n%matplotlib inline\n\n# load saved models\nbasic_cnn = load_model('cats_dogs_basic_cnn.h5')\nimg_aug_cnn = load_model('cats_dogs_cnn_img_aug.h5')\ntl_cnn = load_model('cats_dogs_tlearn_basic_cnn.h5')\ntl_img_aug_cnn = load_model('cats_dogs_tlearn_img_aug_cnn.h5')\ntl_img_aug_finetune_cnn = load_model('cats_dogs_tlearn_finetune_img_aug_cnn.h5')\n\n# load other configurations\nIMG_DIM = (150, 150)\ninput_shape = (150, 150, 3)\nnum2class_label_transformer = lambda l: ['cat' if x == 0 else 'dog' for x in l]\nclass2num_label_transformer = lambda l: [0 if x == 'cat' else 1 for x in l]\n\n# load VGG model for bottleneck features\nfrom keras.applications import vgg16\nfrom keras.models import Model\nimport keras\n\nvgg = vgg16.VGG16(include_top=False, weights='imagenet', \n                  input_shape=input_shape)\noutput = vgg.layers[-1].output\noutput = keras.layers.Flatten()(output)\nvgg_model = Model(vgg.input, output)\nvgg_model.trainable = False\n\ndef get_bottleneck_features(model, input_imgs):\n    features = model.predict(input_imgs, verbose=0)\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_DIM = (150, 150)\n\ntest_files = glob.glob('test_data/*')\ntest_imgs = [img_to_array(load_img(img, target_size=IMG_DIM)) for img in test_files]\ntest_imgs = np.array(test_imgs)\ntest_labels = [fn.split('/')[1].split('.')[0].strip() for fn in test_files]\n\ntest_imgs_scaled = test_imgs.astype('float32')\ntest_imgs_scaled /= 255\ntest_labels_enc = class2num_label_transformer(test_labels)\n\nprint('Test dataset shape:', test_imgs.shape)\nprint(test_labels[0:5], test_labels_enc[0:5])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 1: Basic CNN Performance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = basic_cnn.predict_classes(test_imgs_scaled, verbose=0)\npredictions = num2class_label_transformer(predictions)\ndisplay_model_performance_metrics(true_labels=test_labels, predicted_labels=predictions, \n                                      classes=list(set(test_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 2: Basic CNN with Image Augmentation Performance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = img_aug_cnn.predict_classes(test_imgs_scaled, verbose=0)\npredictions = num2class_label_transformer(predictions)\ndisplay_model_performance_metrics(true_labels=test_labels, predicted_labels=predictions, \n                                      classes=list(set(test_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 3: Transfer Learning — Pre-trained CNN as a Feature Extractor Performance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_bottleneck_features = get_bottleneck_features(vgg_model, test_imgs_scaled)\n\npredictions = tl_cnn.predict_classes(test_bottleneck_features, verbose=0)\npredictions = num2class_label_transformer(predictions)\ndisplay_model_performance_metrics(true_labels=test_labels, predicted_labels=predictions, \n                                      classes=list(set(test_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 4: Transfer Learning — Pre-trained CNN as a Feature Extractor with Image Augmentation Performance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = tl_img_aug_cnn.predict_classes(test_imgs_scaled, verbose=0)\npredictions = num2class_label_transformer(predictions)\ndisplay_model_performance_metrics(true_labels=test_labels, predicted_labels=predictions, \n                                      classes=list(set(test_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 5: Transfer Learning — Pre-trained CNN with Fine-tuning and Image Augmentation Performance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = tl_img_aug_finetune_cnn.predict_classes(test_imgs_scaled, verbose=0)\npredictions = num2class_label_transformer(predictions)\ndisplay_model_performance_metrics(true_labels=test_labels, predicted_labels=predictions, \n                                      classes=list(set(test_labels)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# worst model - basic CNN \nplot_model_roc_curve(basic_cnn, test_imgs_scaled, \n                         true_labels=test_labels_enc, \n                         class_names=[0, 1]) \n\n# best model - transfer learning with fine-tuning & image augmentation \nplot_model_roc_curve(tl_img_aug_finetune_cnn, test_imgs_scaled, \n                         true_labels=test_labels_enc, \n                         class_names=[0, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}