{"cells":[{"metadata":{"_uuid":"025172a23f1afc22da4c086d1d341dbcc5a38847"},"cell_type":"markdown","source":"This kernel uses adapted code and guidance from [deep learning with R](https://www.manning.com/books/deep-learning-with-r) (a fantastic book). It uses a technique called **Gradient-weighted Class Activation Mapping** , the paper of which can be found [here](https://arxiv.org/abs/1610.02391). This is a great technique for sanity-checking which visual elements a CNN is using to make it's classification decisions."},{"metadata":{"trusted":true,"_uuid":"943459b90ebe7e706ee200b5bf5efafcbcae3db5"},"cell_type":"code","source":"#Load the relevant packages,\nsuppressMessages(library(keras))\nsuppressMessages(library(magick))\nsuppressMessages(library(viridis))\n\n#Point to the relevant directories,\nfiles = '../input/dogs-vs-cats-redux-kernels-edition/train/'","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d40a3d7369e1dafdb83fced176d1136d30d7201"},"cell_type":"code","source":"#Load the model. Just use ImageNet weights,\nmodel <- application_xception(\n  weights = \"../input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels.h5\",\n  include_top = TRUE,\n  input_shape = c(224, 224, 3)\n)","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fafa50f8c9b85958b197480a1b5c7b187fe11ce8"},"cell_type":"code","source":"#Choose some images\ntest_images <- paste(files, list.files(files, recursive = T), sep = \"\")","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1bbf2f8c6ca3669e23090a324ab77dbd2d6e575"},"cell_type":"code","source":"set.seed(123)\nsample_test_images = sample(test_images, 10)\nsample_test_images","execution_count":31,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b3c732a18f88f990c75f53def7ee533fbdad070"},"cell_type":"code","source":"#Load image function\nload_and_process = function(img) {\n    img_out = image_load(img, target_size = c(224, 224)) %>%\n    image_to_array() %>%\n    array_reshape(dim = c(1, 224, 224, 3))\n    return(img_out)\n}","execution_count":32,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e9c72e0e54ded2a7b56d2249fab7f61272cc875"},"cell_type":"code","source":"#grad-cam function,\ngrad_cam = function(model, img_out, y, sample_test_images) {\n    preds <- model %>% predict(img_out)\n    max = which.max(preds[1,])\n    test_image_output <- model$output[, max]\n    last_conv_layer <- model %>% get_layer(\"block14_sepconv2_act\") #last layer\n    grads <- k_gradients(test_image_output, last_conv_layer$output)[[1]]\n    pooled_grads <- k_mean(grads, axis = c(1, 2, 3))\n    iterate <- k_function(list(model$input),\n    list(pooled_grads, last_conv_layer$output[1,,,]))\n    c(pooled_grads_value, conv_layer_output_value) %<-% iterate(list(img_out))\n    for (i in 1:512) {\n        conv_layer_output_value[,,i] <-\n        conv_layer_output_value[,,i] * pooled_grads_value[[i]]\n        }\n    heatmap <- apply(conv_layer_output_value, c(1,2), mean)\n    heatmap <- pmax(heatmap, 0)\n    heatmap <- heatmap / max(heatmap)\n    write_heatmap <- function(heatmap, filename, width = 224, height = 224,\n    bg = \"white\", col = terrain.colors(12)) {\n    png(filename, width = width, height = height, bg = bg)\n    op = par(mar = c(0,0,0,0))\n    on.exit({par(op); dev.off()}, add = TRUE)\n    rotate <- function(x) t(apply(x, 2, rev))\n    image(rotate(heatmap), axes = FALSE, asp = 1, col = col)\n    }\n    write_heatmap(heatmap, paste(\"img_heatmap_\", y, \".png\", sep = \"\"))\n    \n    image <- image_read(sample_test_images)\n    info <- image_info(image)\n    geometry <- sprintf(\"%dx%d!\", info$width, info$height)\n    pal <- col2rgb(viridis(20), alpha = TRUE)\n    alpha <- floor(seq(0, 255, length = ncol(pal))) * 0.8\n    pal_col <- rgb(t(pal), alpha = alpha, maxColorValue = 255) \n    write_heatmap(heatmap, paste(\"img_overlay_\", y, \".png\", sep = \"\"),\n    width = 14, height = 14, bg = NA, col = pal_col)\n    return(geometry)\n}","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8865994391f93b5f302b8704f3471cdf6ddc5a02"},"cell_type":"code","source":"#Plot,\npar(mfrow=c(5,2), mar = c(rep(0.1,4)))\n\ny=1\nwhile(y<=length(sample_test_images)) {\n    image_temp = load_and_process(sample_test_images[y])\n    image_temp = image_temp/255\n    geom_temp = grad_cam(model, image_temp, y, sample_test_images[y])\n    image_temp = image_read(paste(\"img_overlay_\", y, \".png\", sep = \"\"))\n    image_temp = image_resize(image_temp, geom_temp, filter = \"quadratic\")\n    image_temp = image_composite(image = image_temp, \n                                 composite_image = image_convert(image_read(sample_test_images[y]), colorspace = 'gray'), \n                                 operator = \"blend\", \n                                 compose_args = \"20\")\n    plot(image_temp)\n    y=y+1\n    }","execution_count":34,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}