{"cells":[{"metadata":{"_uuid":"da492157deb232b8388c80acc31053a62944c1cb"},"cell_type":"markdown","source":"**This kernel is follow up of post on analytics vidhya  [Link](https://www.analyticsvidhya.com/blog/2018/03/essentials-of-deep-learning-visualizing-convolutional-neural-networks/)**"},{"metadata":{"_uuid":"28084f40e7d2d605b8eefe1ebfb7c6d7980a1436"},"cell_type":"markdown","source":"## Preliminary Methods\n### Plotting model architecture"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom scipy.misc import imread\nfrom sklearn.metrics import accuracy_score\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom keras.models import Sequential , Model\nfrom keras.layers import Input, Dense , Flatten , Dropout ,Activation , Conv2D , MaxPooling2D\nimport keras \nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"rn = np.random.RandomState(128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b549bbe1352871bf72772a477fedc423f6fb49a3"},"cell_type":"code","source":"directory = \"../input\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0834f18c32f0e0024090f8c21fc29d315047c816"},"cell_type":"markdown","source":"### loading data "},{"metadata":{"trusted":true,"_uuid":"1330da3ef898a97ebd773fe67e3d5058d46a46a7"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be1143a0533618d347628f86ca62097c4333922b"},"cell_type":"code","source":"train_y = keras.utils.np_utils.to_categorical(train.label.values)\ntrain.drop('label', axis=1 , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40bac8ab06cad2000e53ab432d783be7798ab886"},"cell_type":"code","source":"image_data = train.values\ntrain_x = np.array([image_data[i].reshape(28,28,1) for i in range(image_data.shape[0])])\ntest_image = test.values\ntest_x = np.array([test_image[i].reshape(28,28,1) for i in range(test_image.shape[0])])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"784bdea877e13bb735bdb984b559dbb3cb5855fd"},"cell_type":"code","source":"print(train_y.shape)\nprint(train_x.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df493daaa093d7df785f9d60899c9f028e56fcc8"},"cell_type":"markdown","source":"## Preprocessing Data"},{"metadata":{"trusted":true,"_uuid":"7660a47cc115328671a4d6b8963b8b30f83d11ac"},"cell_type":"code","source":"# Normalising images\n\ntrain_x = train_x/255.0\ntest_x = test_x/255.0\n#train_y = pd.get_dummies(train_y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3420d8598ba27591a42e19ec79228bdbcb078b9"},"cell_type":"markdown","source":"## Splitting dataset\n\n"},{"metadata":{"trusted":true,"_uuid":"30a350052ba89245b1b40cc55f170bc1e2f2a742"},"cell_type":"code","source":"split = int(train_x.shape[0]*0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c39befebdc0c2038ccdfc054d202e929f2cd0e2"},"cell_type":"code","source":"train_x, val_x = train_x[:split], train_x[split:]\ntrain_y, val_y = train_y[:split], train_y[split:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec2517e55ad3a94fcc767bb939a10f2e282475f6"},"cell_type":"markdown","source":"# Creating Model"},{"metadata":{"trusted":true,"_uuid":"f8b4622891a19cfd587c178a67c6160491789694"},"cell_type":"code","source":"epochs =5 \nbatch_size = 128\n\nmodel = Sequential()\nmodel.add(Conv2D(32 , kernel_size=(3,3) , activation='relu' , input_shape=(28,28,1)))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax', name='preds'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40e9b6d4e9cb938267a6204180e9f24b2b76c152"},"cell_type":"code","source":"model.compile(loss='categorical_crossentropy' , optimizer='adam', metrics=['accuracy'])\ntrained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))\npred = model.predict_classes(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5438eac211bb4c1a41f48d67de57fc91a5387e77"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0336f8e70e57e4e775dd87cee72a6f10885ad147"},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58908e7f16173201e8211070b3a6818f41827034"},"cell_type":"code","source":"from IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8169b89563575d555489df49b598a40068ff605e"},"cell_type":"markdown","source":"## Visualize filters"},{"metadata":{"trusted":true,"_uuid":"bd65c3f0b8ab5c45a533924f98f2e7b69b289582"},"cell_type":"code","source":"top_layer = model.layers[0]\nimport matplotlib.pyplot as plt\nplt.imshow(top_layer.get_weights()[0][:, :, :, 0].squeeze(), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2946514f90be62dd33aa3444e5e907d176237dae"},"cell_type":"markdown","source":"## Activation Maps\n### Maximal Activations"},{"metadata":{"_uuid":"deb4abbe3cd2247754e3182b37c84f145d788f6e"},"cell_type":"markdown","source":"**Note: Before going to next part be sure to install keras-vis package. `!pip install keras-vis` would work** \n"},{"metadata":{"trusted":true,"_uuid":"cc6b2517986b23263f56004ca7ab361434dd5829"},"cell_type":"code","source":"!pip uninstall keras-vis\n!pip install git+https://github.com/raghakot/keras-vis.git -U","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6950d29fbf6a948f7f82d5a9459c7d7c91bac6b8"},"cell_type":"code","source":"from vis.visualization import visualize_activation\nfrom vis.utils import utils\nfrom keras import activations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71b79694c66aee9f6387d42e89ad6b1957c8f470"},"cell_type":"code","source":"%matplotlib inline\nplt.rcParams['figure.figsize'] = (18, 6)\n\n# Utility to search for layer index by name.\n# Alternatively we can specify this as -1 since it corresponds to the last layer.\nlayer_idx = utils.find_layer_idx(model, 'preds')\n\n# Swap softmax with linear\nmodel.layers[layer_idx].activation = activations.linear\nmodel = utils.apply_modifications(model)\n\n# This is the output node we want to maximize.\nfilter_idx = 0\nimg = visualize_activation(model, layer_idx, filter_indices=filter_idx)\nplt.imshow(img[..., 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af8619248305a9fce59d00695366d9984c091832","scrolled":false},"cell_type":"code","source":"for output_idx in np.arange(10):\n   # Lets turn off verbose output this time to avoid clutter and just see the output.\n   img = visualize_activation(model, layer_idx, filter_indices=output_idx, input_range=(0., 1.))\n   plt.figure()\n   plt.title('Networks perception of {}'.format(output_idx))\n   plt.imshow(img[..., 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efdea9d6863fe38ea53c6c80fd2f45fc3cf7e1cd"},"cell_type":"markdown","source":"## Image Occlusion\nIn an image classification problem, a natural question is if the model is truly identifying the location of the object in the image, or just using the surrounding context. We took a brief look at this in gradient based methods above. Occlusion based methods attempt to answer this question by systematically occluding different portions of the input image with a grey square, and monitoring the output of the classifier. "},{"metadata":{"trusted":true,"_uuid":"f5893ecfb1511345faebc78a38bf78ebd97069e2"},"cell_type":"code","source":"def iter_occlusion(image, size=8):\n    # taken from https://www.kaggle.com/blargl/simple-occlusion-and-saliency-maps\n\n   occlusion = np.full((size * 5, size * 5, 1), [0.5], np.float32)\n   occlusion_center = np.full((size, size, 1), [0.5], np.float32)\n   occlusion_padding = size * 2\n\n   # print('padding...')\n   image_padded = np.pad(image, ((occlusion_padding, occlusion_padding), (occlusion_padding, occlusion_padding), (0, 0) ), 'constant', constant_values = 0.0)\n\n   for y in range(occlusion_padding, image.shape[0] + occlusion_padding, size):\n\n       for x in range(occlusion_padding, image.shape[1] + occlusion_padding, size):\n           tmp = image_padded.copy()\n\n           tmp[y - occlusion_padding:y + occlusion_center.shape[0] + occlusion_padding, \\\n             x - occlusion_padding:x + occlusion_center.shape[1] + occlusion_padding] \\\n             = occlusion\n\n           tmp[y:y + occlusion_center.shape[0], x:x + occlusion_center.shape[1]] = occlusion_center\n\n           yield x - occlusion_padding, y - occlusion_padding, \\\n             tmp[occlusion_padding:tmp.shape[0] - occlusion_padding, occlusion_padding:tmp.shape[1] - occlusion_padding]\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f4eed3f92a503f5ec06b65e2e76eeca04ffdceb"},"cell_type":"code","source":"train_y[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bb61333a637f60b074ebc846cda0a962e1688e7"},"cell_type":"code","source":"i = 23  # for example\ndata = train_x[i]\ncorrect_class = np.argmax(train_y[4])\n\n# input tensor for model.predict\ninp = data.reshape(1, 28, 28, 1)\n\n# image data for matplotlib's imshow\nimg = data.reshape(28, 28)\n\n# occlusion\nimg_size = img.shape[0]\nocclusion_size = 4\n\n# preview\n_ = plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"396e90cd062e08a89572402c9c7e2fd66bd09c8a"},"cell_type":"code","source":"print('occluding...')\n\nheatmap = np.zeros((img_size, img_size), np.float32)\nclass_pixels = np.zeros((img_size, img_size), np.int16)\n\nfrom collections import defaultdict\ncounters = defaultdict(int)\n\nfor n, (x, y, img_float) in enumerate(iter_occlusion(data, size=occlusion_size)):\n\n    X = img_float.reshape(1, 28, 28, 1)\n    out = model.predict(X)\n    #print('#{}: {} @ {} (correct class: {})'.format(n, np.argmax(out), np.amax(out), out[0][correct_class]))\n    #print('x {} - {} | y {} - {}'.format(x, x + occlusion_size, y, y + occlusion_size))\n\n    heatmap[y:y + occlusion_size, x:x + occlusion_size] = out[0][correct_class]\n    class_pixels[y:y + occlusion_size, x:x + occlusion_size] = np.argmax(out)\n    counters[np.argmax(out)] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3aa4a07fbc701df934659a3bf9e772d86512477b"},"cell_type":"code","source":"pred = model.predict(inp)\nprint('Correct class: {}'.format(correct_class))\nprint('Predicted class: {} (prob: {})'.format(np.argmax(pred), np.amax(out)))\n\nprint('Predictions:')\nfor class_id, count in counters.items():\n    print('{}: {}'.format(class_id, count))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d911988e0619184dab6bb63b985f730950a1a463"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}