{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nfrom fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"bs = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8383ada9a229fe518664321dff7d1e9019371cd"},"cell_type":"code","source":"train=pd.read_csv('../input/labels.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f3ffd440ad072b5060ab9e9a43e2e09065cb5f4"},"cell_type":"code","source":"train.id = train.id+'.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb3d0735bf8b9e12d52d0ed392d4644f92eca765"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4b2d6ef756f46e88bc96f7faae08bd5686ccdaa"},"cell_type":"code","source":"tfms = get_transforms(max_rotate = 20, max_zoom=1.3, max_lighting=0.4, max_warp=0.4, p_affine = 1., p_lighting=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a7a41de94a3bedf8bcf3156d496bcaaa79973a6"},"cell_type":"code","source":"src = (ImageList.from_df(train, '../input/', folder='train')\n      .split_by_rand_pct()\n      .label_from_df(cols = 'breed'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d89cacd64263d85f7d7b2ca1a98b0bbef6fba07"},"cell_type":"code","source":"def get_data(size, bs, padding_mode = 'reflection'):\n    return(src.transform(tfms, size = size, padding_mode=padding_mode)\n          .databunch(bs=bs, num_workers = 0).normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61937b3fb40e0d8eb69c52255a7a82d4b7faec71"},"cell_type":"code","source":"data = get_data(224, bs, 'zeros')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0c7b35c2f735ee8db9e41d3690463ca93d0996d"},"cell_type":"code","source":"def _plot(i,j,ax):\n    x,y = data.train_ds[4]\n    x.show(ax, y=y)\n    \nplot_multi(_plot, 3, 3, figsize=(8,8))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2346b28d3a4117be558b22179bc54cc76e58369"},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true,"_uuid":"1ec4b6efd6a5c0146cad827d672229851eef2a09"},"cell_type":"code","source":"gc.collect()\nlearn = cnn_learner(data, models.resnet34, \n                    metrics = error_rate, bn_final = True, \n                    model_dir = \"/tmp/model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b02b6f3ac97cff4efe8018911dd7f9d6104d02"},"cell_type":"code","source":"learn.fit_one_cycle(3, slice(1e-2), pct_start=0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12133434058a535a160488cef24f7f0779d8d44e"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(2, max_lr=slice(1e-6,1e-3), pct_start=0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fdfb890fae5b1a56ee24bf7672fe9f4d0e6aa7a"},"cell_type":"code","source":"data = get_data(352,bs)\nlearn.data = data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5430fded68b7afc83aedf3cd8e92744397dc3ead"},"cell_type":"code","source":"learn.fit_one_cycle(2, max_lr=slice(1e-6,1e-4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce0f65f219570a6825606b5733b5c416d4b97c8d"},"cell_type":"code","source":"learn.save('352')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32b953ea0098cb30efd50ddbe499c5da38aa2f09"},"cell_type":"markdown","source":"## Convolution kernel"},{"metadata":{"trusted":true,"_uuid":"d9c71afe296444b8c0236f8adce2df6f6078f1ce"},"cell_type":"code","source":"data = get_data(352, 16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba18dd8257dd9f392564380caaa145d7a87e77ca"},"cell_type":"code","source":"learn = cnn_learner(data, models.resnet34, metrics=error_rate, bn_final=True, model_dir = '/tmp/model').load('352')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56a03f3cbbbea1a8ee9491872ce3a61a17084884"},"cell_type":"markdown","source":"Create a convolutional layer by hand. Take a standard 3 x 3 matrix that is going to look for lower right hand cornrers and turn it into a 3d convolutional layer with the .expand command. \n\nCan play around with these convolutinoal layers to see how they interact with the images\n * Make sure to include decimal in first number to make the type of data in kernel floats\n \n [Good Resource for Convolution Visualization](http://setosa.io/ev/image-kernels/)"},{"metadata":{"trusted":true,"_uuid":"e4d58198e1e13263dd5eca75ae187f0b507515ad"},"cell_type":"code","source":"#Right Sobel kernel\nk = tensor ([\n    [-1., 0, 1],\n    [-2., 0, 2],\n    [-1., 0 , 1],\n]).expand(1,3,3,3)/6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27b96fdc783dff16a5cb1de54efa026f41088b81"},"cell_type":"code","source":"#Sharpen Kernel\nk = tensor ([\n    [0., -1, 0],\n    [-1, 5, -1],\n    [0, -1 , 0],\n]).expand(1,3,3,3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6820f2680eadadb5f3150fefe11798bb783c35a9"},"cell_type":"code","source":"k","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c70fdfb591154a3864917888f3b8c9bc203e19a"},"cell_type":"code","source":"k.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5182d831fe3532ff20f4f9171f1cf4fdbdfd51ad"},"cell_type":"code","source":"idx = 2\nt = data.valid_ds[idx][0].data; t.shape #Pull out a single image sample, 0th is image and 1st is label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f9b7d8171b2b1cff9b7ce02bbb529b2a92b8967"},"cell_type":"code","source":"t[None].shape #t[None] is a trick to get a mini-batch of a tensor of size 1, this also works in numpy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"523d628818a45e9144468b5d8033b3bb0923627e"},"cell_type":"code","source":"edge = F.conv2d(t[None], k)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c0e243be20fe400c5ba8c2edb29dd71caa59f56"},"cell_type":"code","source":"show_image(edge[0], figsize = (5,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f478b191c8481cb5de3adb566dd5daecaa03a3c"},"cell_type":"code","source":"x,y = data.valid_ds[idx]\nx.show()\ndata.valid_ds.y[idx]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f34f49b7ecbea3cd265bcc74c5f686590bb8494"},"cell_type":"markdown","source":"This visualization of the convolutinoal layer shows how a convolution can identify an edge/corner, etc"},{"metadata":{"trusted":true,"_uuid":"e53d9b4013b688292abbac9b78b9d6ab5e43c6d8"},"cell_type":"code","source":"#Number of categories\ndata.c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2a5256d83745b99765fd12b78ee2a498ec3628b"},"cell_type":"code","source":"#Details of the resnet model \nlearn.model\n#theres a lot going on in the first Conv layer, but there are 64 chanels and a stride of 2 for the first layer\n#When you stride by 2 you can double the number of chanels (this preserves the complexity of model/memory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8405b202bd0dcafe7326aa568601e7d05fe4a46d"},"cell_type":"code","source":"print(learn.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff8f6e02bd148f2fa275466262a5c362de0be0a7"},"cell_type":"markdown","source":"## Create Heatmap\n\nThe images get boiled down into 11 'sections' of the image through various stride-ings and 512 kernel based channels of for instance (how fluffy is it, how long are ears, etc)\n* So for each 11x11 image, there's an activation for each part of the image for each of the 512 features\n* The network determines itself what are the features based on the optimization of the model\n* If we take the average of all the 512 features we can determine how activated each of the 11x11 parts of the images are"},{"metadata":{"trusted":true,"_uuid":"5b4b61eb83e04e7e6481324f62ca4299476c2afb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"525b605201669bff385f843f5323a5b1b5ad6a09"},"cell_type":"code","source":"m = learn.model.eval();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3847dd89e97e4b56d3b11e74a770e03266e1923f"},"cell_type":"markdown","source":"Before getting heatmap we need to:\n * Put data into pytorch databunch (in this case a single item mini batch)\n * Normalzie the image\n * Put it on the GPU"},{"metadata":{"trusted":true,"_uuid":"e1003cb1c50abc525f4551ede8f770ad1f454f03"},"cell_type":"code","source":"xb,_ = data.one_item(x) #takes all the settings from our previously created data object\nxb_im = Image(data.denorm(xb)[0])\nxb = xb.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1707dae0053d383cf52b62c473bd6d360acc8b9"},"cell_type":"code","source":"from fastai.callbacks.hooks import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"04e6ec16336a13bdbe3c82386450d6f341f54d79"},"cell_type":"markdown","source":"Here we're taking the output of the convolutional layers found in ResNet34, which in the model we've created is found with the m[0] part of the model\n\nhook_output is a fastai module that pulls the output of (in this case m[0]) out of the pytorch back end"},{"metadata":{"trusted":true,"_uuid":"fc5b10d51cf7360536a1bc1cfc8bdc34a257bc78"},"cell_type":"code","source":"??hook_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6394b8a4b0b67d32fafc23b4366fa382f9e7f9e"},"cell_type":"code","source":"def hooked_backward(cat=y):\n    with hook_output(m[0]) as hook_a: #Get activations from the convolution layers\n        with hook_output(m[0], grad = True) as hook_g: #Get gradient from convolution layers\n            preds = m(xb) #DO foreward pass through model\n            preds[0,int(cat)].backward()\n    return hook_a, hook_g","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dce5c5bcb9a31970c7fdcae868118bdcc88320cb"},"cell_type":"code","source":"hook_a, hook_g = hooked_backward()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3200d88f6b1b8e8ea378672f433f3f21243ab3d5"},"cell_type":"code","source":"hook_a.stored","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"612da9601bfe86c2c96824fa23731b6b7dc6854e"},"cell_type":"code","source":"acts = hook_a.stored[0].cpu()\nacts.shape #Now we see our 512 chanels over the 11x11 sections of the image","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f0fb7880b4360e71a3dfe2128d144660013fd0b"},"cell_type":"markdown","source":"Hook allows you to hook into the pytorch machinary itself and run any python code you want"},{"metadata":{"trusted":true,"_uuid":"7ae4ae4c22e348ddede2d2aa6170112c3a393c91"},"cell_type":"code","source":"avg_acts = acts.mean(0)\navg_acts.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4379a6867a32a925f94899edfb737a0a906ccca3"},"cell_type":"code","source":"def show_heatmap(hm):\n    _,ax = plt.subplots()\n    xb_im.show(ax) #fastai function to show the image\n    ax.imshow(hm, alpha = 0.6, extent = (0,352,352, 0), #extent expands the 11x11 image to 352,352\n             interpolation = 'bilinear', cmap = 'magma')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c1ef8da39dba24845cc6634e7bfba32da54ae0a"},"cell_type":"code","source":"show_heatmap(avg_acts)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}