{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# My first Ever Notebook submission \n","metadata":{}},{"cell_type":"markdown","source":"# Cycle-gan : \nThe CycleGAN is a technique that involves the automatic training of image-to-image translation models without paired examples. The models are trained in an unsupervised manner using a collection of images from the source and target domain that do not need to be related in any way.","metadata":{}},{"cell_type":"markdown","source":"## Importing Dependencies \n","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np \nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:12:46.265353Z","iopub.execute_input":"2022-10-02T09:12:46.265770Z","iopub.status.idle":"2022-10-02T09:12:51.136160Z","shell.execute_reply.started":"2022-10-02T09:12:46.265680Z","shell.execute_reply":"2022-10-02T09:12:51.135153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the data ","metadata":{}},{"cell_type":"code","source":"# paths for the data \nmon_path = os.path.join('..','input','gan-getting-started','monet_jpg')\nphoto_path = os.path.join('..','input','gan-getting-started','photo_jpg')\n","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:12:51.138164Z","iopub.execute_input":"2022-10-02T09:12:51.138746Z","iopub.status.idle":"2022-10-02T09:12:51.143506Z","shell.execute_reply.started":"2022-10-02T09:12:51.138714Z","shell.execute_reply":"2022-10-02T09:12:51.142455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helper function to load image\ndef load_image(path,resize=(256,256)):\n    img = tf.keras.utils.load_img(path)\n    img = tf.keras.utils.img_to_array(img)\n    img = tf.image.resize(img,resize)\n    img = (img - 127.5)/127.5 # image values from 0 to 255 -> -1 to 1\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:12:51.144789Z","iopub.execute_input":"2022-10-02T09:12:51.145420Z","iopub.status.idle":"2022-10-02T09:12:51.168302Z","shell.execute_reply.started":"2022-10-02T09:12:51.145378Z","shell.execute_reply":"2022-10-02T09:12:51.167079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"photo_name = os.listdir(photo_path)\nmon_name = os.listdir(mon_path)\nmon_images = np.array([load_image(os.path.join(mon_path,x)) for x in mon_name])\nphoto_images = np.array([load_image(os.path.join(photo_path,x)) for x in photo_name[:300]])\n(mon_images.shape,photo_images.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:12:51.171830Z","iopub.execute_input":"2022-10-02T09:12:51.172263Z","iopub.status.idle":"2022-10-02T09:13:00.423594Z","shell.execute_reply.started":"2022-10-02T09:12:51.172222Z","shell.execute_reply":"2022-10-02T09:13:00.422582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_shape = (256,256,3) # -> (width,height,channel)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:00.426006Z","iopub.execute_input":"2022-10-02T09:13:00.426703Z","iopub.status.idle":"2022-10-02T09:13:00.433703Z","shell.execute_reply.started":"2022-10-02T09:13:00.426660Z","shell.execute_reply":"2022-10-02T09:13:00.431948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GAN \n* We use 2 Generators one for monnet --> photo and other for photo --> monnet\n* We have 4 loss \n    1. Cycle consistency Loss\n    2. Identity loss \n    3. Additional 2 losses for each generator","metadata":{}},{"cell_type":"markdown","source":"## Build Discriminator","metadata":{}},{"cell_type":"code","source":"# define discriminator \ndef build_discriminator(image_shape):\n    #weight initialier \n    init = tf.keras.initializers.RandomNormal(stddev=0.02)\n    \n    input_ = tf.keras.layers.Input(shape=image_shape)\n    d = tf.keras.layers.Conv2D(64,(4,4),strides=(2,2),padding='same',kernel_initializer=init)(input_)\n    d = tf.keras.layers.LeakyReLU(alpha=0.2)(d)\n    \n    d = tf.keras.layers.Conv2D(128,(4,4),strides = (2,2),padding='same',kernel_initializer=init)(d)\n    d = tfa.layers.InstanceNormalization(axis=-1)(d)\n    d = tf.keras.layers.LeakyReLU(alpha=0.2)(d)\n    \n    d = tf.keras.layers.Conv2D(256,(4,4),strides = (2,2),padding='same',kernel_initializer=init)(d)\n    d = tfa.layers.InstanceNormalization(axis=-1)(d)\n    d = tf.keras.layers.LeakyReLU(alpha=0.2)(d)\n    \n    d = tf.keras.layers.Conv2D(512,(4,4),strides = (2,2),padding='same',kernel_initializer=init)(d)\n    d = tfa.layers.InstanceNormalization(axis=-1)(d)\n    d = tf.keras.layers.LeakyReLU(alpha=0.2)(d)\n    \n    d = tf.keras.layers.Conv2D(512,(4,4),padding='same',kernel_initializer=init)(d)\n    d = tfa.layers.InstanceNormalization(axis=-1)(d)\n    d = tf.keras.layers.LeakyReLU(alpha=0.2)(d)\n    \n    patch_out = tf.keras.layers.Conv2D(1,(4,4),padding='same',kernel_initializer = init)(d)\n    \n    model = tf.keras.models.Model(input_,patch_out)\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0002,beta_1=0.5),\n                 loss = 'mse',\n                 loss_weights=[0.5])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:00.435548Z","iopub.execute_input":"2022-10-02T09:13:00.435968Z","iopub.status.idle":"2022-10-02T09:13:00.450377Z","shell.execute_reply.started":"2022-10-02T09:13:00.435927Z","shell.execute_reply":"2022-10-02T09:13:00.449422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Generator\n* We are going to use residual blocks in our generator.","metadata":{}},{"cell_type":"code","source":"# define residual block\ndef residual_block(filters,input_layer):\n    init = tf.keras.initializers.RandomNormal(stddev=0.02)\n    \n    r = tf.keras.layers.Conv2D(filters,(3,3),padding='same',kernel_initializer=init)(input_layer)\n    r = tfa.layers.InstanceNormalization(axis=-1)(r)\n    r = tf.keras.layers.LeakyReLU(alpha=0.2)(r)\n    \n    r = tf.keras.layers.Conv2D(filters,(3,3),padding='same',kernel_initializer=init)(r)\n    r = tfa.layers.InstanceNormalization(axis =-1)(r)\n    \n    r = tf.keras.layers.Concatenate()([r,input_layer])\n    \n    return r","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:00.451734Z","iopub.execute_input":"2022-10-02T09:13:00.452172Z","iopub.status.idle":"2022-10-02T09:13:00.461570Z","shell.execute_reply.started":"2022-10-02T09:13:00.452112Z","shell.execute_reply":"2022-10-02T09:13:00.460678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define generator \ndef build_generator(image_shape,n_resnet=9):\n    init = tf.keras.initializers.RandomNormal(stddev=0.002)\n    \n    input_ = tf.keras.layers.Input(shape=image_shape)\n    \n    g = tf.keras.layers.Conv2D(64,(7,7),padding='same',kernel_initializer=init)(input_)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    g = tf.keras.layers.Activation('relu')(g)\n    \n    g = tf.keras.layers.Conv2D(128,(3,3),strides=(2,2),padding='same',kernel_initializer=init)(g)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    g = tf.keras.layers.Activation('relu')(g)\n    \n    g = tf.keras.layers.Conv2D(128,(3,3),strides=(2,2),padding='same',kernel_initializer=init)(g)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    g = tf.keras.layers.Activation('relu')(g)\n    \n    for _ in range(n_resnet):\n        g = residual_block(256,g)\n        \n    g = tf.keras.layers.Conv2DTranspose(128,(3,3),strides=(2,2),padding='same',kernel_initializer=init)(g)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    g = tf.keras.layers.Activation('relu')(g)\n    \n    g = tf.keras.layers.Conv2DTranspose(64,(3,3),strides=(2,2),padding='same',kernel_initializer=init)(g)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    g = tf.keras.layers.Activation('relu')(g)\n    \n    g = tf.keras.layers.Conv2D(3,(7,7),padding='same',kernel_initializer=init)(g)\n    g = tfa.layers.InstanceNormalization(axis=-1)(g)\n    out_img = tf.keras.layers.Activation('tanh')(g)\n    \n    model = tf.keras.models.Model(input_,out_img)\n    return model\n    ","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:00.462969Z","iopub.execute_input":"2022-10-02T09:13:00.463443Z","iopub.status.idle":"2022-10-02T09:13:00.477593Z","shell.execute_reply.started":"2022-10-02T09:13:00.463410Z","shell.execute_reply":"2022-10-02T09:13:00.476563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = build_generator(image_shape)\nm.build(input_shape=image_shape)\ntf.keras.utils.plot_model(m,show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:00.479333Z","iopub.execute_input":"2022-10-02T09:13:00.479801Z","iopub.status.idle":"2022-10-02T09:13:03.350600Z","shell.execute_reply.started":"2022-10-02T09:13:00.479664Z","shell.execute_reply":"2022-10-02T09:13:03.349217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define composite model\ndef composite_model(g_model_1 , d_model,g_model_2,image_shape):\n    # only set our generative model which we gonna use to transform the image to trainable \n    # rest all model should be non trainable \n    # for eg if we want model to perform photo --> monnet then set that model to trainable in composite model\n    # and set rest of the model to non trainable\n    \n    g_model_1.trainable = True\n    d_model.trainable = False\n    g_model_2.trainable = False\n    \n    #adversial loss\n    input_adl = tf.keras.layers.Input(shape=image_shape)\n    g_model_1_out = g_model_1(input_adl)\n    d_out = d_model(g_model_1_out)\n    \n    # identity loss\n    input_idl = tf.keras.layers.Input(shape=image_shape)\n    g_model_1_out_idl = g_model_1(input_idl) \n    \n    # cycle loss forward \n    out_f = g_model_2(g_model_1_out)\n    # cycle loss backkward\n    g_model_2_out = g_model_2(input_idl)\n    out_b = g_model_1(g_model_2_out)\n    \n    model = tf.keras.models.Model([input_adl,input_idl],[d_out,g_model_1_out_idl,out_f,out_b])\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0002,beta_1=0.5),\n                 loss=['mse','mae','mae','mae'],\n                 loss_weights=[1,5,10,10])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.354193Z","iopub.execute_input":"2022-10-02T09:13:03.354829Z","iopub.status.idle":"2022-10-02T09:13:03.366608Z","shell.execute_reply.started":"2022-10-02T09:13:03.354788Z","shell.execute_reply":"2022-10-02T09:13:03.365343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some Helper Functions ","metadata":{}},{"cell_type":"code","source":"def generate_real_samples(dataset,n_samples,patch_shape):\n    '''\n    Generate real samples to train discriminator\n    dataset: Dataset from where to get real data \n    n_samples : number of samples to take\n    patch_shape : shape of the patch \n    '''\n    ix = np.random.randint(0,dataset.shape[0],n_samples)\n    x = dataset[ix]\n    y = np.ones((n_samples,patch_shape,patch_shape,1))\n    return x,y ","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.367901Z","iopub.execute_input":"2022-10-02T09:13:03.368760Z","iopub.status.idle":"2022-10-02T09:13:03.377954Z","shell.execute_reply.started":"2022-10-02T09:13:03.368725Z","shell.execute_reply":"2022-10-02T09:13:03.377334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_fake_samples(g_model,dataset,patch_shape):\n    '''Generate Fake samples to train discriminator\n    g_model : Generative model to generate fake output\n    dataset : Dataset from which the Generative model takes data for prediction\n    patch_shape : shape of the patch\n    '''\n    x = g_model.predict(dataset)\n    y = np.zeros((len(x),patch_shape,patch_shape,1))\n    return x,y","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.379619Z","iopub.execute_input":"2022-10-02T09:13:03.380790Z","iopub.status.idle":"2022-10-02T09:13:03.387078Z","shell.execute_reply.started":"2022-10-02T09:13:03.380755Z","shell.execute_reply":"2022-10-02T09:13:03.385738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_models(step,g_model_AtoB,g_model_BtoA):\n    # save 1st generative model\n    filename1 = f'g_model_AtoB_{step+1}.h5'\n    g_model_AtoB.save(os.path.join(\"..\",\"save_model\",filename1))\n    # save 2nd generative model\n    filename2 = f'g_model_BtoA_{step+1}.h5'\n    g_model_BtoA.save(os.path.join(\"..\",\"save_model\",filename2))\n    \n    print(f\">>>> Saved : {filename1} {filename2}\")","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.388691Z","iopub.execute_input":"2022-10-02T09:13:03.389863Z","iopub.status.idle":"2022-10-02T09:13:03.398350Z","shell.execute_reply.started":"2022-10-02T09:13:03.389825Z","shell.execute_reply":"2022-10-02T09:13:03.397256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def summarize_performance(step,g_model,x_train,name,n_samples = 5):\n    # select a sample input image\n    x_in ,_ = generate_real_samples(x_train,n_samples,0)\n    # generate translated image\n    x_out , _ = generate_fake_samples(g_model,x_in,0)\n    # scale all pixel value from [-1 to 1 ] to [0 to 1]\n    x_in = (x_in + 1 )/2.0\n    x_out = (x_out + 1)/2.0\n    # plot real image \n    for i in range(n_samples):\n        plt.subplot(2,n_samples,1+i)\n        plt.axis(\"off\")\n        plt.imshow(x_in[i])\n    # plot translated image \n    for i in range(n_samples):\n        plt.subplot(2,n_samples,1+n_samples+i)\n        plt.axis(\"off\")\n        plt.imshow(x_out[i])\n        \n    # save this plot \n    filename1 = f'{name}_generated_plot_{step+1}.png'\n    plt.savefig(os.path.join(\"..\",'eval_plots',filename1))\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.399837Z","iopub.execute_input":"2022-10-02T09:13:03.401051Z","iopub.status.idle":"2022-10-02T09:13:03.411876Z","shell.execute_reply.started":"2022-10-02T09:13:03.401014Z","shell.execute_reply":"2022-10-02T09:13:03.410799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_imagepool(pool,images,max_size=50):\n    selected = list()\n    for image in images:\n        if len(pool)<max_size:\n            #stock the pool \n            pool.append(image)\n            selected.append(image)\n        elif np.random.random() < 0.5:\n            # use image but dont add to the pool \n            selected.append(image)\n        else:\n            #replace existing image and use replaced image\n            ix = np.random.randint(0,len(pool))\n            selected.append(pool[ix])\n            pool[ix] = image\n    return np.asarray(selected)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.413563Z","iopub.execute_input":"2022-10-02T09:13:03.414560Z","iopub.status.idle":"2022-10-02T09:13:03.422633Z","shell.execute_reply.started":"2022-10-02T09:13:03.414524Z","shell.execute_reply":"2022-10-02T09:13:03.421430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Process","metadata":{}},{"cell_type":"code","source":"def train(d_model_A , d_model_B , g_model_AtoB, g_model_BtoA , c_model_AtoB , c_model_BtoA ,dataset,epochs=1):\n    n_epochs , n_batch = epochs,1\n    # determine output sqare shape of the discriminator\n    n_patch = d_model_A.output_shape[1]\n    # load images\n    trainA , trainB = dataset\n    # create pool\n    poolA , poolB = list(),list()\n    # calculate number of batches per training epochs\n    # I read in a research paper that training the model with batch containing only single image \n    # gives better performance\n    batch_per_epoch = int(len(trainA)/n_batch) \n    n_steps = batch_per_epoch * n_epochs\n    \n    for i in range(n_steps):\n        # select set of real samples from domain A and B \n        x_realA , y_realA = generate_real_samples(trainA,n_batch,n_patch)\n        x_realB , y_realB = generate_real_samples(trainB,n_batch,n_patch)\n        # select set of fake samples from domain A and B\n        x_fakeA , y_fakeA = generate_fake_samples(g_model_BtoA,x_realB,n_patch)\n        x_fakeB , y_fakeB = generate_fake_samples(g_model_AtoB,x_realA,n_patch)\n        # update fake images in the pool \n        x_fakeA = update_imagepool(poolA,x_fakeA)\n        x_fakeB = update_imagepool(poolB,x_fakeB)\n        \n        # update  generator b --> a via composite model\n        g_loss_2,_,_,_,_ = c_model_BtoA.train_on_batch([x_realB,x_realA],[y_realA,x_realA,x_realB,x_realA])\n        # update discriminator for a \n        dA_loss1 = d_model_A.train_on_batch(x_realA,y_realA)\n        dA_loss2 = d_model_A.train_on_batch(x_fakeA,y_fakeA)\n        \n        # update generator a --> b via composite model\n        g_loss_1 , _,_,_,_ = c_model_AtoB.train_on_batch([x_realA,x_realB],[y_realB,x_realB,x_realA,x_realB])\n        # update discriminator for b\n        dB_loss1 = d_model_B.train_on_batch(x_realB,y_realB)\n        dB_loss2 = d_model_B.train_on_batch(x_fakeB,y_fakeB)\n         # summarize performance\n        #Since our batch size =1, the number of iterations would be same as the size of our dataset.\n        #In one epoch you'd have iterations equal to the number of images.\n        #If you have 100 images then 1 epoch would be 100 iterations\n        \n        print(f'Iteration======>{i+1}, dA[{dA_loss1}, {dA_loss2}] dB[{dB_loss1} ,{dB_loss2}] g[{g_loss_1}, {g_loss_2}]')\n        # evaluate the model performance periodically\n        #If batch size (total images)=100, performance will be summarized after every 75th iteration.\n#         if (i+1) % (batch_per_epoch * 1) == 0:\n#             # plot A->B translation\n#             summarize_performance(i, g_model_AtoB, trainA, 'AtoB')\n#             # plot B->A translation\n#             summarize_performance(i, g_model_BtoA, trainB, 'BtoA')\n#         if (i+1) % (batch_per_epoch * 5) == 0:\n#             # save the models\n#             # #If batch size (total images)=100, model will be saved after \n#             #every 75th iteration x 5 = 375 iterations.\n#             save_models(i, g_model_AtoB, g_model_BtoA)      ","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.424503Z","iopub.execute_input":"2022-10-02T09:13:03.425451Z","iopub.status.idle":"2022-10-02T09:13:03.437456Z","shell.execute_reply.started":"2022-10-02T09:13:03.425415Z","shell.execute_reply":"2022-10-02T09:13:03.436392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the models\ng_model_AtoB = build_generator(image_shape)\ng_model_BtoA = build_generator(image_shape)\n# discriminators \nd_modelA = build_discriminator(image_shape)\nd_modelB = build_discriminator(image_shape)\n# composite model\nc_model_AtoB = composite_model(g_model_AtoB,d_modelB,g_model_BtoA,image_shape)\nc_model_BtoA = composite_model(g_model_BtoA,d_modelA,g_model_AtoB,image_shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:13:03.438962Z","iopub.execute_input":"2022-10-02T09:13:03.439618Z","iopub.status.idle":"2022-10-02T09:13:11.052655Z","shell.execute_reply.started":"2022-10-02T09:13:03.439582Z","shell.execute_reply":"2022-10-02T09:13:11.050410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train \ntrain(d_modelA,d_modelB,g_model_AtoB,g_model_BtoA,c_model_AtoB,c_model_BtoA,(photo_images,mon_images),epochs=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:20:39.580483Z","iopub.execute_input":"2022-10-02T09:20:39.581434Z","iopub.status.idle":"2022-10-02T09:26:03.364809Z","shell.execute_reply.started":"2022-10-02T09:20:39.581384Z","shell.execute_reply":"2022-10-02T09:26:03.363798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}