{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Create PyTorch Dataloader from object\n# IMPORTANT: Make sure you are in highest directory (i.e. one directory above content)\nimport torch\nimport torchvision\nimport torch.utils.data as data\nfrom torchvision import transforms\nimport PIL\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\n%matplotlib inline\n\nmyTransforms = transforms.Compose([\n    transforms.ColorJitter(hue=.1, saturation=.1),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor()\n])\n!mkdir -p /monet/monet_jpg\n!cp -r /kaggle/input/gan-getting-started/monet_jpg /monet/monet_jpg\nDATA_PATH = \"/monet\"\nBATCH_SIZE = 128\n\nrandom.seed(20)\n\ntrain_data = torchvision.datasets.ImageFolder(root=DATA_PATH, transform = myTransforms)\ntrain_data = torch.utils.data.ConcatDataset([train_data] * 32)\ntrain_data_loader = data.DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Look at i'th image of dataset if you'd like (totally not necessary)\n# May not work if you use a transform with train_data\nfrom IPython.display import display # to display images\ni = 0\n\npil_im = train_data[i][0]\ndisplay(pil_im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function\n#%matplotlib inline\nimport argparse\nimport os\nimport random\nimport torch\nimport torchvision\nimport torch.utils.data as data\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport torchvision.utils as vutils\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\nfrom IPython.display import HTML\n\n\n# Set random seed for reproducibility\nmanualSeed = 999\n#manualSeed = random.randint(1, 10000) # use if you want new results\nprint(\"Random Seed: \", manualSeed)\nrandom.seed(manualSeed)\ntorch.manual_seed(manualSeed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Root directory for dataset\ndataroot = \"~/torch_datasets\"\n\n# Number of workers for dataloader\nworkers = 2\n\n# Batch size during training\nbatch_size = 128\n\n# Spatial size of training images. All images will be resized to this\n#   size using a transformer. MNIST size is actually 28x28 but it should be okay.\n#image_size = 64 # For MNIST\nimage_size = 256 # For Monet\n\n# Number of channels in the training images. For color images this is 3.\n# MNIST is black and white, so 1.\n#nc = 1 # For MNIST\nnc = 3 # For Monet\n\n# Size of z latent vector (i.e. size of generator input)\nnz = 100\n\n# Size of feature maps in generator\n#ngf = 64 # For MNIST\nngf = 32 # For Monet\n\n# Size of feature maps in discriminator\n#ndf = 64 # For MNIST\nndf = 32 # For Monet\n\n# Number of training epochs\nnum_epochs = 50\n\n# Learning rate for optimizers\nlr = 0.0001\n\n# Beta1 hyperparam for Adam optimizers\nbeta1 = 0.5\nbeta2 = 0.5\n\n# Number of GPUs available. Use 0 for CPU mode.\nngpu = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = transforms.Compose([\n                                transforms.Resize(image_size),\n                                transforms.CenterCrop(image_size), \n                                transforms.ToTensor()\n                              ])\n'''\ndataset = dset.MNIST(\n    root=dataroot, train=True, transform=transform, download=True\n)\n'''\ndataset = train_data\n\nloader = torch.utils.data.DataLoader(\n    dataset, batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True\n)\n\nlen(loader.dataset)\n\n# size of mnist image: [1, 64, 64]\n# size of monet image: [3, 256, 256]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"examples = enumerate(loader)\nbatch_idx, (example_data, example_targets) = next(examples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set the device to run on\ndevice = torch.device(\"cuda:0\" if (torch.cuda.is_available() and ngpu > 0) else \"cpu\")\n\n# visualizing the dataset.\nimport matplotlib.pyplot as plt\n\nfig = plt.figure()\nfor i in range(12):\n  plt.subplot(3,4,i+1)\n  plt.tight_layout()\n  # must permute to get shape from [3, 256, 256] -> [256, 256, 3]\n  plt.imshow(example_data[i].permute(1,2,0), interpolation='none') #cmap = 'hsv' maybe\n  plt.xticks([])\n  plt.yticks([])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# custom weights initialization called on netG and netD\ndef weights_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        nn.init.normal_(m.weight.data, 0.0, 0.02)\n    elif classname.find('BatchNorm') != -1:\n        nn.init.normal_(m.weight.data, 1.0, 0.02)\n        nn.init.constant_(m.bias.data, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the Generator\n\nclass Generator(nn.Module):\n    def __init__(self, ngpu):\n        super(Generator, self).__init__()\n        self.ngpu = ngpu\n        self.main = nn.Sequential(\n            # input is Z, going into a convolution\n            nn.ConvTranspose2d( nz, ngf * 8, 10, 1, 0, bias=False),\n            nn.BatchNorm2d(ngf * 8),\n            nn.ReLU(True),\n            # state size. (ngf*8) x 4 x 4\n            nn.ConvTranspose2d(ngf * 8, ngf * 4, 10, 2, 1, bias=False),\n            nn.BatchNorm2d(ngf * 4),\n            nn.ReLU(True),\n            # state size. (ngf*4) x 8 x 8\n            nn.ConvTranspose2d( ngf * 4, ngf * 2, 10, 2, 1, bias=False),\n            nn.BatchNorm2d(ngf * 2),\n            nn.ReLU(True),\n            # \n            nn.ConvTranspose2d( ngf * 2, ngf, 12, 2, 1, bias=False),\n            nn.BatchNorm2d(ngf),\n            nn.ReLU(True),\n            # state size. (ngf) x 32 x 32\n            nn.ConvTranspose2d( ngf, nc, 12, 2, 1, bias=False),\n            nn.Tanh()\n            # state size. (nc) x 64 x 64\n        )\n\n    def forward(self, input):\n        return self.main(input)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate the generator\nnetG = Generator(ngpu).to(device)\n\n# Handle multi-gpu if desired\nif (device.type == 'cuda') and (ngpu > 1):\n    netG = nn.DataParallel(netG, list(range(ngpu)))\n\n# Apply the weights_init function to randomly initialize all weights to mean=0, stdev=0.2.\nnetG.apply(weights_init)\n\n# Print the model\nprint(netG)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Coding the discriminator\nclass Discriminator(nn.Module):\n    def __init__(self, ngpu):\n        super(Discriminator, self).__init__()\n        self.ngpu = ngpu\n        self.main = nn.Sequential(\n            # input is (nc) x 64 x 64\n            nn.Conv2d(nc, ndf, 4, 2, 0, bias=False),\n            nn.LeakyReLU(0.2, inplace=True),\n            # state size. (ndf) x 32 x 32\n            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),\n            nn.BatchNorm2d(ndf * 2),\n            nn.LeakyReLU(0.2, inplace=True),\n            # state size. (ndf*4) x 8 x 8\n            nn.Conv2d(ndf * 2, ndf * 4, 4, 4, 1, bias=False),\n            nn.BatchNorm2d(ndf * 4),\n            nn.LeakyReLU(0.2, inplace=True),\n            # state size. (ndf*4) x 8 x 8\n            nn.Conv2d(ndf * 4, ndf * 8, 4, 4, 1, bias=False),\n            nn.BatchNorm2d(ndf * 8),\n            nn.LeakyReLU(0.2, inplace=True),\n            # state size. (ndf*8) x 4 x 4\n            nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False),\n            nn.Sigmoid()\n        )\n\n    def forward(self, input):\n        return self.main(input)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize the Discriminator\nnetD = Discriminator(ngpu).to(device)\n\n# Handle multi-gpu if desired\nif (device.type == 'cuda') and (ngpu > 1):\n    netD = nn.DataParallel(netD, list(range(ngpu)))\n\n# Apply the weights_init function to randomly initialize all weights to mean=0, stdev=0.2.\nnetD.apply(weights_init)\n\n# Print the model\nprint(netD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize BCELoss function\ncriterion = nn.BCELoss()\n\n# Create batch of latent vectors that we will use to visualize the progression of the generator\nfixed_noise = torch.randn(64, nz, 1, 1, device=device)\n\n# Establish convention for real and fake labels during training\nreal_label = 1.\nfake_label = 0.\n\n# Setup Adam optimizers for both G and D\noptimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, beta2)) # /2.4693701\noptimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, beta2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training Loop\n\n# Lists to keep track of progress\nimg_list = []\nG_losses = []\nD_losses = []\niters = 0\n\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n\nprint(\"Starting Training Loop...\")\n# For each epoch\nfor epoch in range(num_epochs):\n    # For each batch in the dataloader\n    for i, data in enumerate(loader, 0):\n        ############################\n        # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))\n        ###########################\n        ## Train with all-real batch\n        netD.zero_grad()\n        # Format batch\n        print(data[0].shape)\n        real_cpu = data[0].to(device)\n        b_size = real_cpu.size(0)\n        label = torch.full((data[0].shape[0],), real_label, dtype=torch.float, device=device)\n        # Forward pass real batch through D\n        print(\"before output\")\n        output = netD(real_cpu).view(-1) # Getting broken pipe error\n        print(\"after ouput\")\n        print(output.size())\n        # Calculate loss on all-real batch\n        errD_real = criterion(output, label) # This line results in a size mismatch error when I run it with the monet dataset\n        #ValueError: Target and input must have the same number of elements. target nelement (128) != input nelement (21632)\n        # It appears that this is because it is bc the output is 13x13 instead of 1x1\n        # Fixed this, now see broken pipe error\n        # Calculate gradients for D in backward pass\n        errD_real.backward()\n        D_x = output.mean().item()\n\n        ## Train with all-fake batch\n        # Generate batch of latent vectors\n        noise = torch.randn(b_size, nz, 1, 1, device=device)\n        # Generate fake image batch with G\n        fake = netG(noise)\n        label.fill_(fake_label)\n        # Classify all fake batch with D\n        print(fake.detach().shape)\n        output = netD(fake.detach()).view(-1)\n        # Calculate D's loss on the all-fake batch\n        errD_fake = criterion(output, label)\n        # Calculate the gradients for this batch\n        errD_fake.backward()\n        D_G_z1 = output.mean().item()\n        # Add the gradients from the all-real and all-fake batches\n        errD = errD_real + errD_fake\n        # Update D\n        optimizerD.step()\n\n        netG.zero_grad()\n        label.fill_(real_label)  # fake labels are real for generator cost\n        # Since we just updated D, perform another forward pass of all-fake batch through D\n        output = netD(fake).view(-1)\n        # Calculate G's loss based on this output\n        errG = criterion(output, label)\n        # Calculate gradients for G\n        errG.backward()\n        D_G_z2 = output.mean().item()\n        # Update G\n        optimizerG.step()\n\n        # Output training stats\n        #if i % 50 == 0:\n        if True:\n            print('[%d/%d][%d/%d]\\tLoss_D: %.4f\\tLoss_G: %.4f\\tD(x): %.4f\\tD(G(z)): %.4f / %.4f'\n                  % (epoch, num_epochs, i, len(loader),\n                     errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))\n\n        # Save Losses for plotting later\n        G_losses.append(errG.item())\n        D_losses.append(errD.item())\n\n        # Check how the generator is doing by saving G's output on fixed_noise\n        if (iters % 50 == 0) or ((epoch == num_epochs-1) and (i == len(loader)-1)):\n            with torch.no_grad():\n                fake = netG(fixed_noise).detach().cpu()\n            img_list.append(vutils.make_grid(fake, padding=2, normalize=True))\n\n        if D_G_z1 == 0 and D_G_z2 == 0:\n          epoch = num_epochs\n          break\n\n        iters += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title(\"Generator and Discriminator Loss During Training\")\nplt.plot(G_losses,label=\"G\")\nplt.plot(D_losses,label=\"D\")\nplt.xlabel(\"iterations\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(img_list[-1].permute(1,2,0))\nidk = netG(torch.randn(10, nz, 1, 1, device=device)).detach().cpu()\nplt.imshow(np.transpose(idk[1],(1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import ZipFile\nzipObj = ZipFile('images.zip', 'w')\nfor i in range(7000):\n    img = netG(torch.randn(1, nz, 1, 1, device=device)).detach().cpu()[0]\n    fp = 'img' + str(i) + '.jpg'\n    torchvision.utils.save_image(img, fp)\n    zipObj.write(fp)\nzipObj.close()\n!rm *.jpg\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#animation of generator over time\nfig = plt.figure(figsize=(8,8))\nplt.axis(\"off\")\nims = [[plt.imshow(np.transpose(i,(1,2,0)), animated=True)] for i in img_list]\nani = animation.ArtistAnimation(fig, ims, interval=1000, repeat_delay=1000, blit=True)\n\nHTML(ani.to_jshtml())\n\n#plt.imshow(img_list[1].permute(1,2,0))\n#idk = netG(torch.randn(10, nz, 1, 1, device=device)).detach().cpu()\n#plt.imshow(idk[0].permute(1,2,0))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}