{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html\n# https://www.kaggle.com/shihabshahriar/cifar-10-using-pytorch-1-0/data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom copy import deepcopy\nfrom collections import OrderedDict\nimport gc\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import SGD,Adam,lr_scheduler\nfrom torch.utils.data import random_split\nimport torchvision\nfrom torchvision import transforms,models\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!tar -zxvf ../input/cifar10-python/cifar-10-python.tar.gz\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nimg_size = 28 #224\ntrainset_size = 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.Resize(img_size),\n#     transforms.RandomHorizontalFlip(p=.40),\n#     transforms.RandomRotation(30),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntest_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntraindata = torchvision.datasets.CIFAR10(root='.', train=True,download=False, transform=train_transform)\n\ntrainset,valset = random_split(traindata,[trainset_size,50000-trainset_size])\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,shuffle=True)\nvalloader = torch.utils.data.DataLoader(valset, batch_size=batch_size,shuffle=False)\n\ntestset = torchvision.datasets.CIFAR10(root='.', train=False,download=False, transform=test_transform)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=64,shuffle=False)\n\nclasses = ('plane', 'car', 'bird', 'cat','deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = next(iter(trainloader))\nprint(a[0][0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def imshow(img):\n    img = img / 2 + 0.5     # unnormalize\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.show()\nimshow(a[0][0])\nprint(classes[a[1][0]])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom torchvision.utils import make_grid\nimport torchvision.utils as vutils\nimport matplotlib.animation as animation\nfrom IPython.display import HTML\n\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport copy\nimport time\nimport networkx as nx\n\nfrom torch.autograd import Variable\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndevice = 'cpu'\nif torch.cuda.is_available() :\n    device = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# custom weights initialization called on netG and netD\ndef weights_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        nn.init.normal_(m.weight.data, 0.0, 0.02)\n    elif classname.find('BatchNorm') != -1:\n        nn.init.normal_(m.weight.data, 1.0, 0.02)\n        nn.init.constant_(m.bias.data, 0)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Meanwhile, the decision of what computation operation to use sets a particular layer into convolution or average pooling or max pooing.\n\nThe 6 operations available for the controller are:\n\nconvolutions with filter sizes 3 × 3 and 5 × 5,\n\ntorch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros')\n\ndepthwise-separable convolutions with filter sizes 3×3 and 5×5 (Chollet, 2017), \n\nclass depthwise_separable_conv(nn.Module):\n\n    def __init__(self, nin, kernels_per_layer, nout):\n    \n        super(depthwise_separable_conv, self).__init__()\n        \n        self.depthwise = nn.Conv2d(nin, nin * kernels_per_layer, kernel_size=3, padding=1, groups=nin)\n        \n        self.pointwise = nn.Conv2d(nin * kernels_per_layer, nout, kernel_size=1)\n\n    def forward(self, x):\n    \n        out = self.depthwise(x)\n        \n        out = self.pointwise(out)\n        \n        return out\n        \nand max pooling and average pooling of kernel size 3 × 3:\n\ntorch.nn.MaxPool2d(kernel_size, stride=None, padding=0, dilation=1, return_indices=False, ceil_mode=False)\n\ntorch.nn.AvgPool2d(kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None)\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_max_pool_layer(ksize):\n    return nn.MaxPool2d(ksize).apply(weights_init)\n\n\ndef get_avg_pool_layer(ksize):\n    return nn.AvgPool2d(ksize).apply(weights_init)\n\ndef get_conv_layer(in_lyr,out_lyr,ksize):\n    seq_layer = nn.Sequential(\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_lyr, out_lyr, ksize),\n            nn.BatchNorm2d(out_lyr),\n    )\n    return seq_layer.apply(weights_init)\n\nclass depthwise_separable_conv(nn.Module):\n    def __init__(self, nin, kernels_per_layer, nout):\n        super(depthwise_separable_conv, self).__init__()\n        self.depthwise = nn.Conv2d(nin, nin * kernels_per_layer, kernel_size=3, padding=1, groups=nin).apply(weights_init)\n        self.pointwise = nn.Conv2d(nin * kernels_per_layer, nout, kernel_size=1).apply(weights_init)\n\n    def forward(self, x):\n        out = self.depthwise(x)\n        out = self.pointwise(out)\n        return out\n\ndef get_sep_layer(in_lyr,out_lyr,ksize):\n    return depthwise_separable_conv(in_lyr,ksize,out_lyr)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1. Conv 3\n# 2. Conv 5\n# 3. Sep 3\n# 4. Sep 5\n# 5. MaxPool\n# 6. AvgPool\n\nout_features_shape = [10,24,32,64,128]\noutput_size = 10\nnum_actvn_fns = 6\n\ndef get_new_layer(actvn,x):\n    ofsindex = 1\n    actvn = actvn -1\n#     print(\"Actvn -- \"+str(actvn))\n    \n    out_features_shape[ofsindex] = x.shape[1]\n    \n    if actvn == 0:\n        return get_conv_layer(x.shape[1],out_features_shape[ofsindex],3).to(device)\n    if actvn == 1:\n        return get_conv_layer(x.shape[1],out_features_shape[ofsindex],5).to(device)\n    if actvn == 2:\n        return get_sep_layer(x.shape[1],out_features_shape[ofsindex],3).to(device)\n    if actvn == 3:\n        return get_sep_layer(x.shape[1],out_features_shape[ofsindex],5).to(device)\n    if actvn == 4:\n        return get_max_pool_layer(3).to(device)\n    if actvn == 5:\n        return get_avg_pool_layer(3).to(device)\n    return get_avg_pool_layer(3).to(device)\n\ndef get_out_layer(size):\n    nlayer = nn.Sequential(\n            nn.Linear(in_features=size[1]*size[2]*size[3], out_features=10),\n            nn.LogSoftmax(),\n        )\n    return nlayer.apply(weights_init).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def concatzeroes(max_height,max_width,b):\n    dim = 2\n    pad_size_1 = int((max_height-b.shape[dim]) / 2)\n    pad_size_2 = (max_height-b.shape[dim]) - pad_size_1\n    b = F.pad(input=b, pad=(0, 0, pad_size_1, pad_size_2), mode='constant', value=0)\n\n    dim = 3\n    pad_size_1 = int((max_width-b.shape[dim]) / 2)\n    pad_size_2 = (max_width-b.shape[dim]) - pad_size_1\n    b = F.pad(input=b, pad=(pad_size_1, pad_size_2,0,0), mode='constant', value=0)\n    \n    return b\n\ndef downsizetensors(max_height,max_width,b):\n    size = (max_height,max_width)\n    return F.interpolate(b, size=size, mode='bilinear', align_corners=False)\n\ndef concatenate2(a,b,increase_size):\n    if(increase_size):\n        max_height = max(a.shape[2],b.shape[2])\n        max_width = max(a.shape[3],b.shape[3])\n        a = concatzeroes(max_height,max_width,a)\n        b = concatzeroes(max_height,max_width,b)\n        return torch.cat((a, b), 1)\n    else:\n        max_height = min(a.shape[2],b.shape[2])\n        max_width = min(a.shape[3],b.shape[3])\n        a = downsizetensors(max_height,max_width,a)\n        b = downsizetensors(max_height,max_width,b)\n        return torch.cat((a, b), 1)\n\n\ndef concatenate3(a,b,c,increase_size):\n    if(increase_size):\n        max_height = max(a.shape[2],b.shape[2],c.shape[2])\n        max_width = max(a.shape[3],b.shape[3],c.shape[3])\n        a = concatzeroes(max_height,max_width,a)\n        b = concatzeroes(max_height,max_width,b)\n        c = concatzeroes(max_height,max_width,c)\n        return torch.cat((torch.cat((a, b), 1), c), 1)\n    else:\n        \n        max_height = min(a.shape[2],b.shape[2],c.shape[2])\n        max_width = min(a.shape[3],b.shape[3],c.shape[3])\n        a = downsizetensors(max_height,max_width,a)\n        b = downsizetensors(max_height,max_width,b)\n        c = downsizetensors(max_height,max_width,c)\n        return torch.cat((torch.cat((a, b), 1), c), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DAG(nn.Module):\n    \n    def __init__(self,num_layers):\n        super(DAG, self).__init__()\n        self.num_layers = num_layers\n        self.myparameters = nn.ParameterList()\n        self.dag = {}\n        self.dag_calculated = {}\n        self.loose_ends = []\n        self.first_run = 0\n        self.config = \"0\"\n    def get_key(self,lnum):\n        nconfig = self.config[0:int(3 * (lnum-1) )+1]\n        key = \"\"\n        for c in nconfig:\n            key+=str(c)\n        return key\n\n    def initialize_param_grads(self):\n        for p in self.myparameters:\n            p.requires_grad = True\n#         for lnum in range(1,self.num_layers+1):\n# #             print(self.dag[get_key(lnum,config)])\n#             for p in nn.ParameterList(self.dag[get_key(lnum,config)].parameters()):\n#                 p.requires_grad = True\n#         out_layer_key = get_key(self.num_layers,config)+\"9\"\n#         for p in nn.ParameterList(self.dag[out_layer_key].parameters()):\n#             p.requires_grad = True\n        return\n\n        \n    #fac == forward acc to config\n    def fac(self,x):\n        self.loose_ends = []\n        for i in range(1,self.num_layers+1):\n            self.loose_ends.append(i)\n            \n        layer_activation = int(self.config[0])\n        key = self.get_key(1)\n        \n        if key not in self.dag.keys():\n            layer_t = get_new_layer(layer_activation,x)\n            self.myparameters += nn.ParameterList(layer_t.parameters())\n            self.dag[key] = layer_t\n            for p in nn.ParameterList(layer_t.parameters()):\n                p.requires_grad = True\n\n            \n        x = self.dag[key](x)\n        self.dag_calculated[key] = x\n#         print(x.shape)\n        \n        for lnum in range(2,self.num_layers+1):\n#             print(lnum)\n            key = self.get_key(lnum)\n            prev_layer_1 = int(self.config[3*(lnum-1)+1 -3])\n            prev_layer_2 = int(self.config[3*(lnum-1)+2 -3])\n            layer_activation = int(self.config[3*(lnum-1)])\n            \n            if prev_layer_1 > lnum or prev_layer_2 > lnum:\n                return None\n            \n            prev_layer_1_key = self.get_key(prev_layer_1)\n            prev_layer_2_key = self.get_key(prev_layer_2)\n            \n            \n            if key not in self.dag.keys():\n                layer_t = get_new_layer(layer_activation,x)\n                self.myparameters += nn.ParameterList(layer_t.parameters())\n                self.dag[key] = layer_t\n                for p in nn.ParameterList(layer_t.parameters()):\n                    p.requires_grad = True\n            x = self.dag[key](x)\n            if prev_layer_1_key != prev_layer_2_key:\n                x = concatenate3(self.dag_calculated[prev_layer_1_key],self.dag_calculated[prev_layer_2_key],x,0)\n            else:\n                x = concatenate2(self.dag_calculated[prev_layer_1_key],x,0)\n            self.dag_calculated[key] = x\n            \n            \n#             print(self.loose_ends)\n            if self.loose_ends!=None and prev_layer_1 in self.loose_ends:\n                self.loose_ends.remove(prev_layer_1)\n            if self.loose_ends!=None and prev_layer_2 in self.loose_ends:\n                self.loose_ends.remove(prev_layer_2)\n#             print(self.loose_ends)\n            \n        num_loose_ends = len(self.loose_ends)\n        if(num_loose_ends<=0):\n            return None\n        \n        out = self.dag_calculated[self.get_key(self.loose_ends[0])]\n        for i in range(1,num_loose_ends):\n            out = concatenate2(out,self.dag_calculated[self.get_key(self.loose_ends[i])],0)\n        \n        out_layer_key = self.get_key(self.num_layers)+\"9\"\n        if out_layer_key not in self.dag.keys():\n            layer_t= get_out_layer(out.shape)\n            self.dag[out_layer_key] = layer_t\n            self.myparameters += nn.ParameterList(layer_t.parameters())\n#         print(out.shape)\n        out = out.view(out.shape[0],-1)\n        self.dag_calculated[out_layer_key] = self.dag[out_layer_key](out)\n        return self.dag_calculated[out_layer_key]\n\n    def forward(self, x,config=None):\n        return self.fac(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_nodes = 6\ndag_model = DAG(num_nodes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 0.0002\n# Initialize BCELoss function\ncriterion = nn.NLLLoss()\noptimizer = None\n\n\ndef model_train_loop(cnn_config):\n    dag_model.config = cnn_config\n    dag_model.initialize_param_grads()\n    for epoch in range(10):\n        # For each batch in the dataloader\n        for i, data in enumerate(trainloader, 0):\n            images, labels = data\n            images, labels = images.to(device),labels.to(device)\n            output = dag_model(images)\n            if i==0 and epoch==0:\n                optimizer = optim.Adam(dag_model.myparameters, lr=lr, betas=(0.5, 0.999))\n            dag_model.zero_grad()\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n#         print(loss)\n#             if i%100==0:\n#                 print(((torch.sum(torch.argmax(output,1) == labels)).float()/labels.shape[0]).item())\n#     #             print(loss.item())\n        \n    return Variable(((torch.sum(torch.argmax(output,1) == labels)).float()/labels.shape[0]), requires_grad=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_train_loop([2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 4, 1, 4, 4, 5, 4])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Controller Part ToDo : \n1. Define LSTM that generates 3*num_nodes -2 integers in the expected ranges and create configurations.\n\n2. Write cost function and Back Propogation for the controller\n\nnn.LSTM(input_dim, hidden_dim, n_layers, batch_first=True)\n\nhttps://stackoverflow.com/questions/48302810/whats-the-difference-between-hidden-and-output-in-pytorch-lstm\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.distributions import Categorical\nfrom torch.autograd import Variable","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"controller_num_epochs = 1\ncontroller_optimizer = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class Controller(nn.Module):\n\n#     def __init__(self):\n#         super(Controller, self).__init__()\n#         self.n_layers = 3\n#         self.generated_config = []\n#         self.policy_history = []\n\n#         self.lstm1 = nn.LSTM(num_actvn_fns, num_nodes ,self.n_layers).to(device)\n#         self.hidden1 = (torch.randn(self.n_layers, 1, num_nodes).to(device),torch.randn(self.n_layers, 1, num_nodes).to(device))\n#         self.prev_out1 = torch.randn(1,1,num_nodes).to(device)\n\n#         self.lstm2 = nn.LSTM(num_nodes, num_nodes ,self.n_layers).to(device)\n#         self.hidden2 = (torch.randn(self.n_layers, 1, num_nodes).to(device),torch.randn(self.n_layers, 1, num_nodes).to(device))\n#         self.prev_out2 = torch.randn(1, 1, num_nodes).to(device)\n        \n#         self.lstm3 = nn.LSTM(num_nodes, num_actvn_fns ,self.n_layers).to(device)\n#         self.hidden3 = (torch.randn(self.n_layers, 1, num_actvn_fns).to(device),torch.randn(self.n_layers, 1, num_actvn_fns).to(device))\n#         self.prev_out3 = torch.randn(1, 1, num_actvn_fns).to(device)\n        \n        \n        \n#     def update_controller(self,rewards):\n#         self.zero_grad()\n#         rewards = torch.Tensor(rewards)\n#         loss = (torch.sum(torch.mul(torch.Tensor(self.policy_history), Variable(rewards,requires_grad=True)).mul(-1), -1))\n#         loss.backward(retain_graph=True)\n#         controller_optimizer.step()\n#         print(loss)\n#         return loss\n    \n#     def get_reward(self):\n#         lent = len(self.generated_config)\n#         lnum = int(lent/3) + 1\n#         rewards = [0]*lent\n#         if self.generated_config[lent-1] >= lnum :\n#             rewards[lent-1] = 1000\n#             self.update_controller(rewards)\n#             return 1\n# #         self.update_controller(rewards)\n#         return 0\n    \n    \n#     def forward(self):\n#         self.generated_config = []\n#         self.policy_history = []\n#         self.zero_grad()\n#         rewards = []\n#         for i in range(3*num_nodes-2):\n#             if i%3 ==0:\n#                 out, self.hidden1 = self.lstm1(self.prev_out3, self.hidden1)\n#                 out = F.sigmoid(out)\n#                 self.prev_out1 = out\n#                 probs = Categorical(out)\n#                 action = probs.sample()\n#                 self.policy_history.append(Variable(probs.log_prob(action),requires_grad=True))\n#                 self.generated_config.append(action.item())\n#             if i%3 ==1:\n#                 out, self.hidden2 = self.lstm2(self.prev_out1, self.hidden2)\n#                 out = F.sigmoid(out)\n#                 self.prev_out2 = out\n#                 probs = Categorical(out)\n#                 action = probs.sample()\n#                 self.policy_history.append(Variable(probs.log_prob(action),requires_grad=True))\n#                 self.generated_config.append(torch.argmax(out).item())\n#                 if self.get_reward():\n#                     return None\n#                 return\n#             if i%3 ==2:\n#                 out, self.hidden3 = self.lstm3(self.prev_out2, self.hidden3)\n#                 out = F.sigmoid(out)\n#                 self.prev_out3 = out\n#                 probs = Categorical(out)\n#                 action = probs.sample()\n#                 self.policy_history.append(Variable(probs.log_prob(action),requires_grad=True))\n#                 self.generated_config.append(torch.argmax(out).item())\n#                 if self.get_reward():\n#                     return None\n#         return self.generated_config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Controller(nn.Module):\n\n    def __init__(self):\n        super(Controller, self).__init__()\n        self.n_layers = 3\n        self.generated_config = []\n        self.policy_history = []\n\n        self.lstm = nn.LSTM(num_actvn_fns, num_nodes ,self.n_layers).to(device)\n        self.hidden = (torch.randn(self.n_layers, 1, num_nodes).to(device),torch.randn(self.n_layers, 1, num_nodes).to(device))\n        self.prev_out = torch.randn(1,1,num_nodes).to(device)\n\n    def update_controller(self,rewards):\n        self.zero_grad()\n        rewards = torch.Tensor(rewards)\n        loss = (torch.sum(torch.mul(torch.Tensor(self.policy_history), Variable(rewards,requires_grad=True)).mul(-1), -1))\n        loss.backward(retain_graph=True)\n        controller_optimizer.step()\n#         print(loss)\n        return loss\n    \n    def get_reward(self):\n        lent = len(self.generated_config)\n        lnum = int(lent/3) + 1\n        rewards = [0]*lent\n        if (lent-1)%3 == 0:\n            if self.generated_config[lent-1] == 0 or self.generated_config[lent-1] > 6:\n                rewards[lent-1] = -1\n                self.update_controller(rewards)\n                return 1\n        if (lent-2)%3 == 0:\n            if self.generated_config[lent-1] == 0 or self.generated_config[lent-1] >= lnum:\n                rewards[lent-1] = -1\n                self.update_controller(rewards)\n                return 1\n        if (lent)%3 == 0:\n            if self.generated_config[lent-1] == 0 or self.generated_config[lent-1] >= lnum:\n                rewards[lent-1] = -1\n                self.update_controller(rewards)\n                return 1\n#         self.update_controller(rewards)\n        return 0\n    \n    \n    def forward(self):\n        self.generated_config = []\n        self.policy_history = []\n        self.zero_grad()\n        rewards = []\n        for i in range(3*num_nodes-2):\n            out, self.hidden = self.lstm(self.prev_out, self.hidden)\n            out = F.sigmoid(out)\n            self.prev_out = out\n            probs = Categorical(out)\n            action = probs.sample()\n            self.policy_history.append(Variable(probs.log_prob(action),requires_grad=True))\n            self.generated_config.append(torch.argmax(out).item())\n            if self.get_reward():\n                return None\n        return self.generated_config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"controller_model = Controller().to(device)\ncontroller_optimizer = optim.Adam(controller_model.parameters(), lr=0.1, betas=(0.5, 0.999))\ncontroller_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"controller_num_epochs = 100000\nfor controller_epoch in range(controller_num_epochs):\n    config = controller_model()\n    if config!=None:\n        print(config)\n        print(model_train_loop(config))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}