{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport zipfile\nimport glob\nfrom PIL import Image\nimport copy\nimport random\nfrom time import time\nfrom typing import Any, Dict\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import optim\nfrom torch.utils.data import Dataset, TensorDataset, DataLoader\nimport torchvision\nimport torchvision.transforms as transforms\nfrom matplotlib import pyplot as plt\nfrom IPython.display import HTML\nimport matplotlib.animation as animation","metadata":{"execution":{"iopub.status.busy":"2022-02-17T07:12:30.320646Z","iopub.execute_input":"2022-02-17T07:12:30.321061Z","iopub.status.idle":"2022-02-17T07:12:31.817107Z","shell.execute_reply.started":"2022-02-17T07:12:30.320983Z","shell.execute_reply":"2022-02-17T07:12:31.816305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0. Import Data","metadata":{"execution":{"iopub.status.busy":"2022-02-15T03:49:02.628895Z","iopub.execute_input":"2022-02-15T03:49:02.629443Z","iopub.status.idle":"2022-02-15T03:49:15.030293Z","shell.execute_reply.started":"2022-02-15T03:49:02.62941Z","shell.execute_reply":"2022-02-15T03:49:15.029335Z"}}},{"cell_type":"code","source":"# Unzip the dataset\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as z:\n    z.extractall(\".\")\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\", \"r\") as z:\n    z.extractall(\".\")\n# Make Data directory (limited to dog data)\n!mkdir -p images/dog\n!mv train/dog.*.jpg images/dog/","metadata":{"execution":{"iopub.status.busy":"2022-02-17T07:12:31.818973Z","iopub.execute_input":"2022-02-17T07:12:31.819238Z","iopub.status.idle":"2022-02-17T07:12:49.419787Z","shell.execute_reply.started":"2022-02-17T07:12:31.819190Z","shell.execute_reply":"2022-02-17T07:12:49.418934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make data set with resized and normalized\nimage_size = 64\ndataset = torchvision.datasets.ImageFolder(\n    root = \"images\", \n    transform = transforms.Compose(\n        [transforms.Resize((image_size, image_size)),\n         transforms.CenterCrop(image_size),\n         transforms.ToTensor(),\n         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]))","metadata":{"execution":{"iopub.status.busy":"2022-02-17T07:12:49.421346Z","iopub.execute_input":"2022-02-17T07:12:49.421627Z","iopub.status.idle":"2022-02-17T07:12:49.495945Z","shell.execute_reply.started":"2022-02-17T07:12:49.421588Z","shell.execute_reply":"2022-02-17T07:12:49.495286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Constract DCGAN Model","metadata":{"execution":{"iopub.status.busy":"2022-02-15T04:13:02.132415Z","iopub.execute_input":"2022-02-15T04:13:02.132773Z","iopub.status.idle":"2022-02-15T04:13:02.492418Z","shell.execute_reply.started":"2022-02-15T04:13:02.132738Z","shell.execute_reply":"2022-02-15T04:13:02.491764Z"}}},{"cell_type":"markdown","source":"Refer to https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html","metadata":{"execution":{"iopub.status.busy":"2022-02-15T04:51:34.844664Z","iopub.execute_input":"2022-02-15T04:51:34.845014Z","iopub.status.idle":"2022-02-15T04:51:34.851689Z","shell.execute_reply.started":"2022-02-15T04:51:34.844978Z","shell.execute_reply":"2022-02-15T04:51:34.850435Z"}}},{"cell_type":"markdown","source":"## 1.1. Generator","metadata":{}},{"cell_type":"code","source":"# Set latent vector size\nn_z = 100\n# Set image size\nn_f = 64\n# Set　channel size\nn_c = 3\nclass Generator(nn.Module):\n    def __init__(self, n_z, n_f, n_c, relu_slope=0.2):\n        super().__init__()\n        \n        self.network = nn.Sequential(nn.ConvTranspose2d(in_channels=n_z, out_channels=n_f*8, kernel_size=4, stride=1, padding=0, bias=False),\n                                     nn.BatchNorm2d(n_f*8),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.ConvTranspose2d(in_channels=n_f*8, out_channels=n_f*4, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f*4),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.ConvTranspose2d(in_channels=n_f*4, out_channels=n_f*2, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f*2),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.ConvTranspose2d(in_channels=n_f*2, out_channels=n_f, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.ConvTranspose2d(in_channels=n_f, out_channels=n_c, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.Tanh()\n                                    )\n    def forward(self, x):\n        return self.network(x)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:05.790083Z","iopub.execute_input":"2022-02-15T08:12:05.790445Z","iopub.status.idle":"2022-02-15T08:12:05.804134Z","shell.execute_reply.started":"2022-02-15T08:12:05.790388Z","shell.execute_reply":"2022-02-15T08:12:05.803063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2. Discriminator","metadata":{}},{"cell_type":"code","source":"class Discriminator(nn.Module):\n    def __init__(self, n_f, n_c, relu_slope=0.2):\n        super().__init__()\n        self.network = nn.Sequential(nn.Conv2d(in_channels=n_c, out_channels=n_f, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.Conv2d(in_channels=n_f, out_channels=n_f*2, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f*2),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.Conv2d(in_channels=n_f*2, out_channels=n_f*4, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f*4),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.Conv2d(in_channels=n_f*4, out_channels=n_f*8, kernel_size=4, stride=2, padding=1, bias=False),\n                                     nn.BatchNorm2d(n_f*8),\n                                     nn.LeakyReLU(negative_slope=relu_slope,inplace=True),\n                                     nn.Conv2d(in_channels=n_f*8, out_channels=1, kernel_size=4, stride=1, padding=0, bias=False),\n                                     nn.Sigmoid()\n                                    )\n    def forward(self, x):\n        return self.network(x)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:05.806175Z","iopub.execute_input":"2022-02-15T08:12:05.806461Z","iopub.status.idle":"2022-02-15T08:12:05.824466Z","shell.execute_reply.started":"2022-02-15T08:12:05.806387Z","shell.execute_reply":"2022-02-15T08:12:05.823416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print model outline\ngen = Generator(n_z, n_f, n_c)\nprint('Generator================')\nprint(gen)\ndis = Discriminator(n_f, n_c)\nprint('Discriminator================')\nprint(dis)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:05.82563Z","iopub.execute_input":"2022-02-15T08:12:05.825889Z","iopub.status.idle":"2022-02-15T08:12:05.953548Z","shell.execute_reply.started":"2022-02-15T08:12:05.825851Z","shell.execute_reply":"2022-02-15T08:12:05.952614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3. Constract weights initialization func","metadata":{}},{"cell_type":"code","source":"def weights_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        nn.init.normal_(m.weight.data, 0.0, 0.02)\n    elif classname.find('BatchNorm') != -1:\n        nn.init.normal_(m.weight.data, 1.0, 0.02)\n        nn.init.constant_(m.bias.data, 0)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:05.954922Z","iopub.execute_input":"2022-02-15T08:12:05.955265Z","iopub.status.idle":"2022-02-15T08:12:05.961848Z","shell.execute_reply.started":"2022-02-15T08:12:05.955235Z","shell.execute_reply":"2022-02-15T08:12:05.960698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Learning","metadata":{}},{"cell_type":"code","source":"# Lock random seed============================\ntorch.manual_seed(20220215)\ntorch.cuda.manual_seed(20220215)\ntorch.backends.cudnn.deterministic = True\ntorch.use_deterministic_algorithms = True\n\n# Allocate device to use GPU============================\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Create models============================\ndis = Discriminator(n_f, n_c).to(device)\ngen = Generator(n_z, n_f, n_c).to(device)\n# Initialize weights============================\ndis.apply(weights_init)\ngen.apply(weights_init)\n\n# Create latent vectors used to visualize\nfixed_noise = torch.randn(image_size, n_z, 1, 1, device=device)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:05.963507Z","iopub.execute_input":"2022-02-15T08:12:05.964205Z","iopub.status.idle":"2022-02-15T08:12:06.102668Z","shell.execute_reply.started":"2022-02-15T08:12:05.964156Z","shell.execute_reply":"2022-02-15T08:12:06.101715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.1. Create mini-batch data","metadata":{}},{"cell_type":"code","source":"batch_size = 32\ndataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:29:59.705213Z","iopub.execute_input":"2022-02-15T08:29:59.706209Z","iopub.status.idle":"2022-02-15T08:29:59.711542Z","shell.execute_reply.started":"2022-02-15T08:29:59.706156Z","shell.execute_reply":"2022-02-15T08:29:59.710646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2. Iteration","metadata":{}},{"cell_type":"code","source":"# Learning rate=================\nlr = 0.0001\n\n# Set loss func=================\ncriterion = nn.BCELoss()\n\n# Set optimizer=================\noptimizer_d = optim.Adam(dis.parameters(), lr=lr, betas=(0.5, 0.999))\noptimizer_g = optim.Adam(gen.parameters(), lr=lr, betas=(0.5, 0.999))\n\n# Set epochs=================\nnum_epochs = 20\n\nimg_list = []\ng_loss = []\nd_loss = []\niters = 0\n\n## Iteration=================\nfor epoch in range(num_epochs):\n    for i, batch in enumerate(dataloader):\n        \n        # (1) Update Discriminator===============================================\n        dis.train()\n        gen.train()\n        X, _ = batch\n        X = X.to(device)\n        dis.zero_grad()\n        # create labels (all 1) for Discriminator\n        label = torch.full((X.shape[0],),1.,dtype=torch.float,device=device)\n        # Predictions for real images by D (i.e., [0, 1])\n        output = dis(X).view(-1)\n        # Calculate loss\n        errD_real = criterion(output, label)\n        # Calculate gradients for Discriminator\n        errD_real.backward()\n        D_x = output.mean().item()\n\n        ## Train with all-fake batch\n        # Generate batch of latent vectors\n        noise = torch.randn(batch_size, n_z, 1, 1, device=device)\n        # Generate fake image batch with G\n        fake = gen(noise)\n        label.fill_(0.)\n        # Classify all fake batch with Discriminator\n        output = dis(fake.detach()).view(-1)\n        # Calculate Discriminator's loss on the all-fake batch\n        errD_fake = criterion(output, label)\n        # Calculate the gradients for this batch, accumulated (summed) with previous gradients\n        errD_fake.backward()\n        D_G_z1 = output.mean().item()\n        # Compute error of D as sum over the fake and the real batches\n        errD = errD_real + errD_fake\n        # Update D\n        optimizer_d.step()\n\n        # (2) Update Generator===============================================\n        gen.zero_grad()\n        label.fill_(1)  # fake labels are real for generator cost\n        # Since we just updated D, perform another forward pass of all-fake batch through D\n        output = dis(fake).view(-1)\n        # Calculate Generator's loss based on this output\n        errG = criterion(output, label)\n        # Calculate gradients for Generator\n        errG.backward()\n        D_G_z2 = output.mean().item()\n        # Update G\n        optimizer_g.step()\n\n        # Output training stats\n        if i % 50 == 0:\n            print('[%d/%d][%d/%d]\\tLoss_D: %.4f\\tLoss_G: %.4f\\tD(x): %.4f\\tD(G(z)): %.4f / %.4f'\n                  % (epoch, num_epochs, i, len(dataloader),\n                     errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))\n\n        # Save Losses for plotting later\n        g_loss.append(errG.item())\n        d_loss.append(errD.item())\n\n        # Check how the generator is doing by saving G's output on fixed_noise\n        dis.eval()\n        gen.eval()\n\n    # Save generated images for each epoch\n    with torch.no_grad():\n        fake = gen(fixed_noise).detach().cpu()\n    img_list.append(torchvision.utils.make_grid(fake, padding=2, normalize=True))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:12:43.052599Z","iopub.execute_input":"2022-02-15T08:12:43.052903Z","iopub.status.idle":"2022-02-15T08:21:49.099033Z","shell.execute_reply.started":"2022-02-15T08:12:43.052871Z","shell.execute_reply":"2022-02-15T08:21:49.097937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Result","metadata":{}},{"cell_type":"markdown","source":"## 3.1. Visualize Training loss","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:59:09.748314Z","iopub.execute_input":"2022-02-15T07:59:09.748594Z","iopub.status.idle":"2022-02-15T07:59:09.765657Z","shell.execute_reply.started":"2022-02-15T07:59:09.748567Z","shell.execute_reply":"2022-02-15T07:59:09.764179Z"}}},{"cell_type":"code","source":"# Visulize loss\nfig, axes = plt.subplots(1,1,figsize=(10,6))\naxes.plot(d_loss,label=\"Discriminator\",color='green')\naxes.plot(g_loss,label=\"Generator\",color='orange')\naxes.set_title('Loss curve',size=16)\naxes.set_xlabel('Iterations',size=16)\naxes.set_ylabel('Loss',size=16)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:25:53.314873Z","iopub.execute_input":"2022-02-15T08:25:53.315231Z","iopub.status.idle":"2022-02-15T08:25:53.663338Z","shell.execute_reply.started":"2022-02-15T08:25:53.315196Z","shell.execute_reply":"2022-02-15T08:25:53.647504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2. Visualize generated images","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(8,8))\nplt.axis(\"off\")\nims = [[plt.imshow(np.transpose(i,(1,2,0)), animated=True)] for i in img_list]\nani = animation.ArtistAnimation(fig, ims, interval=1000, repeat_delay=1000, blit=True)\n\nHTML(ani.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:26:35.152771Z","iopub.execute_input":"2022-02-15T08:26:35.153173Z","iopub.status.idle":"2022-02-15T08:26:35.724606Z","shell.execute_reply.started":"2022-02-15T08:26:35.153136Z","shell.execute_reply":"2022-02-15T08:26:35.723922Z"},"trusted":true},"execution_count":null,"outputs":[]}]}