{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import torch\nimport pretrainedmodels\nfrom torch.autograd import Function\nfrom torchvision import models\nfrom torchvision import utils\nimport cv2\nimport sys\nimport numpy as np\nimport argparse\nimport os\ni=0\n\nfrom torch import nn\n\n\nimage = []\nclass FeatureExtractor():\n#     \"\"\" Class for extracting activations and\n#     registering gradients from targetted intermediate layers \"\"\"\n    def __init__(self, model, target_layers):\n        self.model = model\n        self.target_layers = target_layers\n        self.gradients = []\n    def save_gradient(self, grad):\n        self.gradients.append(grad)\n    def __call__(self, x):\n        outputs = []\n        self.gradients = []\n        for name, module in self.model._modules.items():\n            print('name=',name)\n            print('x.size()=',x.size())\n            x = module(x)\n            if name in self.target_layers:\n                x.register_hook(self.save_gradient)\n                outputs += [x]\n        return outputs, x\n    \nclass ModelOutputs():\n#      \"\"\" Class for making a forward pass, and getting:\n#      1. The network output.\n#      2. Activations from intermeddiate targetted layers.\n#      3. Gradients from intermeddiate targetted layers. \"\"\"\n    def __init__(self, model, target_layers,use_cuda):\n        self.model = model\n        self.feature_extractor = FeatureExtractor(self.model, target_layers)\n        self.cuda = use_cuda\n    def get_gradients(self):\n        return self.feature_extractor.gradients\n    def __call__(self, x):\n        target_activations, output  = self.feature_extractor(x)\n        output = output.view(output.size(0), -1)\n        print('classfier=',output.size())\n        if self.cuda:\n            output = resnet_or.last_linear(output).cuda()\n        else:\n            output = resnet_or.last_linear(output)\n        return target_activations, output\n    \n    \ndef preprocess_image(img):\n    means = [0.485, 0.456, 0.406]\n    stds = [0.229, 0.224, 0.225]\n    preprocessed_img = img.copy()[: , :, ::-1]\n    for i in range(3):\n        preprocessed_img[:, :, i] = preprocessed_img[:, :, i] - means[i]\n        preprocessed_img[:, :, i] = preprocessed_img[:, :, i] / stds[i]\n    preprocessed_img = np.ascontiguousarray(np.transpose(preprocessed_img, (2, 0, 1)))\n    preprocessed_img = torch.from_numpy(preprocessed_img)\n    preprocessed_img.unsqueeze_(0)\n    input = torch.Tensor(preprocessed_img)\n    return input\n    \ndef show_cam_on_image(img, mask,name):\n    heatmap = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)\n    heatmap = np.float32(heatmap) / 255\n    cam = heatmap + np.float32(img)\n    cam = cam / np.max(cam)\n    concat = np.concatenate((img, cam), axis=1)\n    cv2.imwrite(\"./grad_cam/grad_{}.jpg\".format(name), np.uint8(255 * concat))\n\nclass GradCam:\n    def __init__(self, model, target_layer_names, use_cuda):\n        self.model = model\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n        self.extractor = ModelOutputs(self.model, target_layer_names, use_cuda)\n        \n    def forward(self, input):\n          return self.model(input)\n    \n    def __call__(self, input, index = None):\n        if self.cuda:\n            features, output = self.extractor(input.cuda())\n        else:\n            features, output = self.extractor(input)\n        if index == None:\n            index = np.argmax(output.cpu().data.numpy())\n        one_hot = np.zeros((1, output.size()[-1]), dtype = np.float32)\n        one_hot[0][index] = 1\n        one_hot = torch.Tensor(torch.from_numpy(one_hot))\n        if self.cuda:\n            one_hot = torch.sum(one_hot.cuda() * output)\n        else:\n            one_hot = torch.sum(one_hot * output)\n        self.model.zero_grad()\n        one_hot.backward(retain_graph=True)\n        grads_val = self.extractor.get_gradients()[-1].cpu().data.numpy()\n  \n        target = features[-1]\n        target = target.cpu().data.numpy()[0, :]\n         \n        weights = np.mean(grads_val, axis = (2, 3))[0, :]\n        cam = np.zeros(target.shape[1 : ], dtype = np.float32)\n        for i, w in enumerate(weights):\n            cam += w * target[i, :, :]\n        cam = np.maximum(cam, 0)\n        cam = cv2.resize(cam, (224, 224))\n        cam = cam - np.min(cam)\n        cam = cam / np.max(cam)\n        return cam\n    \nclass GuidedBackpropReLU(Function):\n    def forward(self, input):\n        positive_mask = (input > 0).type_as(input)\n        output = torch.addcmul(torch.zeros(input.size()).type_as(input), input, positive_mask)\n        self.save_for_backward(input, output)\n        return output\n    def backward(self, grad_output):\n        input, output = self.saved_tensors\n        grad_input = None\n        positive_mask_1 = (input > 0).type_as(grad_output)\n        positive_mask_2 = (grad_output > 0).type_as(grad_output)\n        grad_input = torch.addcmul(torch.zeros(input.size()).type_as(input), torch.addcmul(torch.zeros(input.size()).type_as(input), grad_output, positive_mask_1), positive_mask_2)\n        return grad_input\n    \nclass GuidedBackpropReLUModel:\n    def __init__(self, model, use_cuda):\n        self.model = resnet\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n\n        for idx, module in self.model._modules.items():\n           if module.__class__.__name__ == 'ReLU':\n                self.model._modules[idx] = GuidedBackpropReLU()\n                \n    def forward(self, input):\n        return self.model(input)\n    \n    def __call__(self, input, index = None):\n        if self.cuda:\n            output = self.forward(input.cuda())\n        else:\n            output = self.forward(input)\n        if index == None:\n            index = np.argmax(output.cpu().data.numpy())\n\n        one_hot = np.zeros((1, output.size()[-1]), dtype = np.float32)\n        one_hot[0][index] = 1\n        one_hot = torch.Tensor(torch.from_numpy(one_hot))\n        if self.cuda:\n            one_hot = torch.sum(one_hot.cuda() * output)\n        else:\n            one_hot = torch.sum(one_hot * output)\n        one_hot.backward(retain_graph=True)\n        output = input.grad.cuda().data.numpy()\n        output = output[0,:,:,:]\n        return output\n    \n\nif __name__ == '__main__':\n    #  \"\"\" python grad_cam.py <path_to_image>\n    #  1. Loads an image with opencv.\n    #  2. Preprocesses it for VGG19 and converts to a pytorch variable.\n    #  3. Makes a forward pass to find the category index with the highest score,\n    #  and computes intermediate activations.\n    #  Makes the visualization. \"\"\"\n\n    model_name = 'resnet50'\n    res50 = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')\n    num_ftrs = res50.last_linear.in_features\n    res50.last_linear = nn.Linear(num_ftrs, 196)\n    model = res50\n    model.load_state_dict(torch.load('./weight_best.pt'))\n    #print(model)\n\n    del model.last_linear\n    # Can work with any model, but it assumes that the model has a\n    # feature method, and a classifier method,\n    # as in the VGG models in torchvision.\n\n    grad_cam = GradCam(model, target_layer_names = [\"layer4\"], use_cuda=True)\n    image_path = './test_crop/'\n    x=os.walk(image_path)\n    for root,dirs,filename in x:\n\n        pass\n    for s in filename:\n        image.append(cv2.imread(image_path+s,1))\n\n    for img in image:\n        img = np.float32(cv2.resize(img, (224, 224))) / 255\n        input = preprocess_image(img)\n        print('input.size()=',input.size())\n        # If None, returns the map for the highest scoring category.\n        # Otherwise, targets the requested index.\n        target_index = None\n        mask = grad_cam(input, target_index)\n        #print(type(mask))\n        i=i+1\n        show_cam_on_image(img, mask,i)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}