{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as neural_network\nimport torch.nn.functional as functions\nimport torchvision.transforms as transforms\nimport torch.nn as model\nimport torch.nn.functional as functions\nimport torch.optim as optim\n\nfrom torchvision import datasets","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":1.529306,"end_time":"2021-01-22T17:14:53.169622","exception":false,"start_time":"2021-01-22T17:14:51.640316","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-22T18:00:14.862005Z","iopub.execute_input":"2022-11-22T18:00:14.862611Z","iopub.status.idle":"2022-11-22T18:00:17.184503Z","shell.execute_reply.started":"2022-11-22T18:00:14.862479Z","shell.execute_reply":"2022-11-22T18:00:17.183069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating Digits using GANs","metadata":{"papermill":{"duration":0.012018,"end_time":"2021-01-22T17:14:53.194288","exception":false,"start_time":"2021-01-22T17:14:53.182270","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def download_data_digits():\n    return datasets.MNIST(root='data', train=True, download=True, transform=transforms.ToTensor())\n\ntrain_data = download_data_digits()","metadata":{"papermill":{"duration":3.103718,"end_time":"2021-01-22T17:14:56.334481","exception":false,"start_time":"2021-01-22T17:14:53.230763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-22T18:00:17.186509Z","iopub.execute_input":"2022-11-22T18:00:17.187130Z","iopub.status.idle":"2022-11-22T18:00:19.231732Z","shell.execute_reply.started":"2022-11-22T18:00:17.187090Z","shell.execute_reply":"2022-11-22T18:00:19.230615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2022-11-22T18:00:46.011179Z","iopub.execute_input":"2022-11-22T18:00:46.011660Z","iopub.status.idle":"2022-11-22T18:00:46.021947Z","shell.execute_reply.started":"2022-11-22T18:00:46.011621Z","shell.execute_reply":"2022-11-22T18:00:46.020437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Data Loader","metadata":{"papermill":{"duration":0.015693,"end_time":"2021-01-22T17:14:56.367715","exception":false,"start_time":"2021-01-22T17:14:56.352022","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = torch.utils.data.DataLoader(train_data, batch_size=64, num_workers=2)","metadata":{"papermill":{"duration":0.022651,"end_time":"2021-01-22T17:14:56.406152","exception":false,"start_time":"2021-01-22T17:14:56.383501","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:39.146352Z","iopub.execute_input":"2022-11-19T09:14:39.147502Z","iopub.status.idle":"2022-11-19T09:14:39.155838Z","shell.execute_reply.started":"2022-11-19T09:14:39.147460Z","shell.execute_reply":"2022-11-19T09:14:39.154600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizating Samples Test","metadata":{"papermill":{"duration":0.015654,"end_time":"2021-01-22T17:14:56.437769","exception":false,"start_time":"2021-01-22T17:14:56.422115","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def show_samples_test(length=10):\n    images, labels = iter(train).next()\n    images = images.numpy()\n    \n    indexes = 0\n    for k in range(0, 5):\n        fig, axs = plt.subplots(1, length, figsize=(10, 10))\n\n        for i in range(0, length):\n            axs[i].imshow(np.squeeze(images[i + indexes]), label=labels[i + i + indexes])\n            axs[i].set_title('Y: {}'.format(labels[i + indexes]))\n            leg = axs[i].legend()\n            \n        indexes += 10\n","metadata":{"papermill":{"duration":0.025876,"end_time":"2021-01-22T17:14:56.479653","exception":false,"start_time":"2021-01-22T17:14:56.453777","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:39.532664Z","iopub.execute_input":"2022-11-19T09:14:39.533893Z","iopub.status.idle":"2022-11-19T09:14:39.542960Z","shell.execute_reply.started":"2022-11-19T09:14:39.533848Z","shell.execute_reply":"2022-11-19T09:14:39.541784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_samples_test(10)","metadata":{"papermill":{"duration":5.113851,"end_time":"2021-01-22T17:15:01.609625","exception":false,"start_time":"2021-01-22T17:14:56.495774","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:39.731876Z","iopub.execute_input":"2022-11-19T09:14:39.732368Z","iopub.status.idle":"2022-11-19T09:14:44.803738Z","shell.execute_reply.started":"2022-11-19T09:14:39.732326Z","shell.execute_reply":"2022-11-19T09:14:44.802625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Discriminator Class","metadata":{"papermill":{"duration":0.022392,"end_time":"2021-01-22T17:15:01.654563","exception":false,"start_time":"2021-01-22T17:15:01.632171","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Discriminator(neural_network.Module):\n\n    def __init__(self):\n        super(Discriminator, self).__init__()\n        \n        self.input_layer    = neural_network.Linear(784, 128)\n        self.hidden_layer_1 = neural_network.Linear(128, 64)\n        self.hidden_layer_2 = neural_network.Linear(64, 32)\n        self.output_layer   = neural_network.Linear(32, 1)\n\n        self.dropout = neural_network.Dropout(0.3)\n        \n        \n    def forward(self, img_input):\n        img_input = img_input.view(-1, 28 * 28)\n        img_input = functions.leaky_relu(self.input_layer(img_input), 0.2) \n        img_input = self.dropout(img_input)\n        img_input = functions.leaky_relu(self.hidden_layer_1(img_input), 0.2)\n        img_input = self.dropout(img_input)\n        img_input = functions.leaky_relu(self.hidden_layer_2(img_input), 0.2)\n        img_input = self.dropout(img_input)\n        \n        return self.output_layer(img_input)\n","metadata":{"papermill":{"duration":0.034241,"end_time":"2021-01-22T17:15:01.710363","exception":false,"start_time":"2021-01-22T17:15:01.676122","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.805809Z","iopub.execute_input":"2022-11-19T09:14:44.806134Z","iopub.status.idle":"2022-11-19T09:14:44.815277Z","shell.execute_reply.started":"2022-11-19T09:14:44.806105Z","shell.execute_reply":"2022-11-19T09:14:44.814057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generator Class","metadata":{"papermill":{"duration":0.020886,"end_time":"2021-01-22T17:15:01.752171","exception":false,"start_time":"2021-01-22T17:15:01.731285","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Generator(neural_network.Module):\n\n    def __init__(self):\n        super(Generator, self).__init__()\n\n        self.input_layer    =  neural_network.Linear(100, 32)\n        self.hidden_layer_1 =  neural_network.Linear(32, 64)\n        self.hidden_layer_2 =  neural_network.Linear(64, 128)\n        self.output_layer   =  neural_network.Linear(128, 784)\n        \n        self.dropout = neural_network.Dropout(0.3)\n\n    def forward(self, img_input):\n        img_input = functions.leaky_relu(self.input_layer(img_input), 0.2)\n        img_input = self.dropout(img_input)\n        img_input = functions.leaky_relu(self.hidden_layer_1(img_input), 0.2)\n        img_input = self.dropout(img_input)\n        img_input = functions.leaky_relu(self.hidden_layer_2(img_input), 0.2)\n        img_input = self.dropout(img_input)\n        \n        return functions.tanh(self.output_layer(img_input))\n","metadata":{"papermill":{"duration":0.032691,"end_time":"2021-01-22T17:15:01.805763","exception":false,"start_time":"2021-01-22T17:15:01.773072","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.816614Z","iopub.execute_input":"2022-11-19T09:14:44.817111Z","iopub.status.idle":"2022-11-19T09:14:44.832127Z","shell.execute_reply.started":"2022-11-19T09:14:44.817065Z","shell.execute_reply":"2022-11-19T09:14:44.831174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D = Discriminator()\nG = Generator()","metadata":{"papermill":{"duration":0.029421,"end_time":"2021-01-22T17:15:01.856484","exception":false,"start_time":"2021-01-22T17:15:01.827063","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.834140Z","iopub.execute_input":"2022-11-19T09:14:44.834494Z","iopub.status.idle":"2022-11-19T09:14:44.849402Z","shell.execute_reply.started":"2022-11-19T09:14:44.834427Z","shell.execute_reply":"2022-11-19T09:14:44.848603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def real_loss(D_out):\n    batch_size = D_out.size(0)\n    criterion = neural_network.BCEWithLogitsLoss()\n    return criterion(D_out.squeeze(), torch.ones(batch_size) * 0.95)\n    \n\ndef fake_loss(D_out):\n    criterion = neural_network.BCEWithLogitsLoss()\n    return criterion(D_out.squeeze(), torch.zeros(D_out.size(0)))","metadata":{"papermill":{"duration":0.030053,"end_time":"2021-01-22T17:15:01.958162","exception":false,"start_time":"2021-01-22T17:15:01.928109","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.860192Z","iopub.execute_input":"2022-11-19T09:14:44.860543Z","iopub.status.idle":"2022-11-19T09:14:44.868366Z","shell.execute_reply.started":"2022-11-19T09:14:44.860512Z","shell.execute_reply":"2022-11-19T09:14:44.867252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_optimizer = optim.Adam(D.parameters(), 0.002)\ng_optimizer = optim.Adam(G.parameters(), 0.002)","metadata":{"papermill":{"duration":0.028787,"end_time":"2021-01-22T17:15:02.008582","exception":false,"start_time":"2021-01-22T17:15:01.979795","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.870011Z","iopub.execute_input":"2022-11-19T09:14:44.870331Z","iopub.status.idle":"2022-11-19T09:14:44.878600Z","shell.execute_reply.started":"2022-11-19T09:14:44.870296Z","shell.execute_reply":"2022-11-19T09:14:44.877425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GAN Training\n\n## Train Discriminator\n","metadata":{"papermill":{"duration":0.022106,"end_time":"2021-01-22T17:15:02.052167","exception":false,"start_time":"2021-01-22T17:15:02.030061","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_discriminator(G, D, optimizer, real_data, batch_size, size): \n\n    optimizer.zero_grad()\n    fake_data = G(random_vector(batch_size, size))\n    total_loss = real_loss(D(real_data)) + fake_loss(D(fake_data))\n    total_loss.backward()\n    optimizer.step()\n    \n    return total_loss","metadata":{"papermill":{"duration":0.030355,"end_time":"2021-01-22T17:15:02.104206","exception":false,"start_time":"2021-01-22T17:15:02.073851","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.880159Z","iopub.execute_input":"2022-11-19T09:14:44.880608Z","iopub.status.idle":"2022-11-19T09:14:44.889327Z","shell.execute_reply.started":"2022-11-19T09:14:44.880575Z","shell.execute_reply":"2022-11-19T09:14:44.888479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Generator","metadata":{"papermill":{"duration":0.021491,"end_time":"2021-01-22T17:15:02.147434","exception":false,"start_time":"2021-01-22T17:15:02.125943","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ndef train_generator(G, D, optimizer, batch_size, size):\n    optimizer.zero_grad()\n    fake_data = G(random_vector(batch_size, size))\n    loss = real_loss(D(fake_data))\n    loss.backward()\n    optimizer.step()\n    return loss","metadata":{"papermill":{"duration":0.030107,"end_time":"2021-01-22T17:15:02.199261","exception":false,"start_time":"2021-01-22T17:15:02.169154","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.890421Z","iopub.execute_input":"2022-11-19T09:14:44.891078Z","iopub.status.idle":"2022-11-19T09:14:44.900846Z","shell.execute_reply.started":"2022-11-19T09:14:44.891043Z","shell.execute_reply":"2022-11-19T09:14:44.899989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_vector(batch_size, lenx):\n    return torch.from_numpy(np.random.uniform(-1, 1, size=(batch_size, lenx))).float()","metadata":{"papermill":{"duration":0.028642,"end_time":"2021-01-22T17:15:02.250043","exception":false,"start_time":"2021-01-22T17:15:02.221401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.904834Z","iopub.execute_input":"2022-11-19T09:14:44.905696Z","iopub.status.idle":"2022-11-19T09:14:44.912831Z","shell.execute_reply.started":"2022-11-19T09:14:44.905659Z","shell.execute_reply":"2022-11-19T09:14:44.912012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_training_sample(data):\n    fig, axes = plt.subplots(figsize=(7,7), nrows=4, ncols=4)\n    for ax, img in zip(axes.flatten(), data[-1]):\n        img = img.detach()\n        ax.imshow(img.reshape((28,28)))\n    plt.show()\n    \n    \ndef show_training_loss(d_loss_, g_loss):\n    plt.plot(d_loss_arr, label=\"N.Discriminator\", alpha=0.5)\n    plt.plot(g_loss_arr, label=\"N.Generator\", alpha=0.5)\n    plt.title(\"Trainings loss\")\n    plt.legend()\n    plt.show()\n    ","metadata":{"papermill":{"duration":0.033203,"end_time":"2021-01-22T17:15:02.305264","exception":false,"start_time":"2021-01-22T17:15:02.272061","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.913888Z","iopub.execute_input":"2022-11-19T09:14:44.914640Z","iopub.status.idle":"2022-11-19T09:14:44.923558Z","shell.execute_reply.started":"2022-11-19T09:14:44.914605Z","shell.execute_reply":"2022-11-19T09:14:44.922508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle as pkl\n\nmax_epochs = 300\n\nsamples = list()\nlosses =  list()\nd_loss_arr = list()\ng_loss_arr = list()\n\nZ = torch.from_numpy(np.random.uniform(-1, 1, size=(16, 100))).float()\n\nD.train()\nG.train()\n                           \nfor epoch in range(1, max_epochs):\n    for index, (img, l) in enumerate(train):\n        img = img * 2 - 1 \n        d_loss = train_discriminator(G, D, d_optimizer, img, 64, size=100)\n        g_loss = train_generator(G, D, g_optimizer, 64, 100)\n        \n        d_loss_arr.append(d_loss.item())\n        g_loss_arr.append(g_loss.item())\n        \n      \n        if (epoch%3 == 0) and (index % 1200 == 0 or index == 1):\n            print(f'[{epoch}] -- discriminator_loss: {d_loss.item()}  -- generator_loss: {g_loss.item()}')\n        \n    \n    losses.append((d_loss.item(), g_loss.item()))\n\n    G.eval() \n    samples.append(G(Z))\n    G.train()\n    if epoch%3 == 0:\n        show_training_sample(samples)\n        show_training_loss(d_loss_arr, g_loss_arr)","metadata":{"papermill":{"duration":1719.268399,"end_time":"2021-01-22T17:43:41.595829","exception":false,"start_time":"2021-01-22T17:15:02.327430","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-19T09:14:44.924748Z","iopub.execute_input":"2022-11-19T09:14:44.925036Z","iopub.status.idle":"2022-11-19T10:24:42.056882Z","shell.execute_reply.started":"2022-11-19T09:14:44.925009Z","shell.execute_reply":"2022-11-19T10:24:42.055664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}