{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":33252,"databundleVersionId":2977712,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install utils","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:40:46.770353Z","iopub.execute_input":"2024-10-16T13:40:46.770783Z","iopub.status.idle":"2024-10-16T13:41:00.32594Z","shell.execute_reply.started":"2024-10-16T13:40:46.770742Z","shell.execute_reply":"2024-10-16T13:41:00.32457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\nimport tensorflow as tf\n\nimport scipy\nfrom scipy.io import loadmat\nimport re\n\nimport string\nimport imageio\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\nimport time\nimport nltk\nfrom utils import*\n\nfrom matplotlib.pyplot import figure\nfrom PIL import Image\n\nimport cv2\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:03.470491Z","iopub.execute_input":"2024-10-16T13:41:03.470961Z","iopub.status.idle":"2024-10-16T13:41:03.47961Z","shell.execute_reply.started":"2024-10-16T13:41:03.470917Z","shell.execute_reply":"2024-10-16T13:41:03.478206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dictionary_path = '/kaggle/input/datalab-cup3-reverse-image-caption-2021/dictionary'\nvocab = np.load(dictionary_path + '/vocab.npy')\nword2Id_dict = dict(np.load(dictionary_path + '/word2Id.npy'))\nid2word_dict = dict(np.load(dictionary_path + '/id2Word.npy'))","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:05.019812Z","iopub.execute_input":"2024-10-16T13:41:05.020294Z","iopub.status.idle":"2024-10-16T13:41:05.071086Z","shell.execute_reply.started":"2024-10-16T13:41:05.020252Z","shell.execute_reply":"2024-10-16T13:41:05.069942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sent2IdList(line, MAX_SEQ_LENGTH=20):\n    MAX_SEQ_LIMIT = MAX_SEQ_LENGTH\n    padding = 0\n    prep_line = re.sub('[%s]' % re.escape(string.punctuation), ' ', line.rstrip())\n    prep_line = prep_line.replace('-', ' ')\n    prep_line = prep_line.replace('-', ' ')\n    prep_line = prep_line.replace('  ', ' ')\n    prep_line = prep_line.replace('.', '')\n    tokens = prep_line.split(' ')\n    tokens = [\n        tokens[i] for i in range(len(tokens))\n        if tokens[i] != ' ' and tokens[i] != ''\n        ]\n    l = len(tokens)\n    padding = MAX_SEQ_LIMIT - l\n    for i in range(padding):\n        tokens.append('<PAD>')\n        line = [\n          word2Id_dict[tokens[k]]\n          if tokens[k] in word2Id_dict else word2Id_dict['<RARE>']\n          for k in range(len(tokens))\n        ]\n\n    return line","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:06.250075Z","iopub.execute_input":"2024-10-16T13:41:06.251051Z","iopub.status.idle":"2024-10-16T13:41:06.260274Z","shell.execute_reply.started":"2024-10-16T13:41:06.251004Z","shell.execute_reply":"2024-10-16T13:41:06.259118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Load the dataset\ndata_path = '/kaggle/input/datalab-cup3-reverse-image-caption-2021/dataset'\ndf = pd.read_pickle(data_path + '/text2ImgData.pkl')\n\n# Sort the DataFrame by 'ID' column in place\ndf.sort_values(by='ID', inplace=True)\n\n# Check the number of training samples\nnum_training_sample = len(df)\nn_images_train = num_training_sample\n\n# Display the first 5 rows of the sorted DataFrame\nprint(df.head(5))\n","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:07.755222Z","iopub.execute_input":"2024-10-16T13:41:07.755673Z","iopub.status.idle":"2024-10-16T13:41:11.441003Z","shell.execute_reply.started":"2024-10-16T13:41:07.755632Z","shell.execute_reply":"2024-10-16T13:41:11.439775Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter and print rows where the 'ID' is between 6730 and 6740 (inclusive)\nfiltered_df = df[(df.index >= 6730) & (df.index <= 6740)]\n\n# Display the filtered rows\nprint(filtered_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:17.575183Z","iopub.execute_input":"2024-10-16T13:41:17.575617Z","iopub.status.idle":"2024-10-16T13:41:17.613936Z","shell.execute_reply.started":"2024-10-16T13:41:17.575578Z","shell.execute_reply":"2024-10-16T13:41:17.612631Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_HEIGHT = 64\nIMAGE_WIDTH = 64\nIMAGE_CHANNEL = 3\n\ndef training_data_generator(caption, image_path):\n    # load in the image according to image path\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_image(img, channels=3)\n    img = tf.image.convert_image_dtype(img, tf.float32)\n    img.set_shape([None, None, 3])\n    img = tf.image.resize(img, size=[IMAGE_HEIGHT, IMAGE_WIDTH])\n    img.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNEL])\n    caption = tf.cast(caption, tf.int32)\n\n    return img, caption\n\ndef dataset_generator(filenames, batch_size, data_generator):\n    # load the training data into two NumPy arrays\n    df = pd.read_pickle(filenames)\n    captions = df['Captions'].values\n    caption = []\n    # each image has 1 to 10 corresponding captions\n    # we choose one of them randomly for training\n    for i in range(len(captions)):\n        caption.append(random.choice(captions[i]))\n    caption = np.asarray(caption)\n    caption = caption.astype(np.int32)\n    image_path = df['ImagePath'].values\n    \n    # assume that each row of `features` corresponds to the same row as `labels`.\n    assert caption.shape[0] == image_path.shape[0]\n    \n    dataset = tf.data.Dataset.from_tensor_slices((caption, image_path))\n    dataset = dataset.map(data_generator, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    dataset = dataset.shuffle(len(caption)).batch(batch_size, drop_remainder=True)\n    dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:22.135503Z","iopub.execute_input":"2024-10-16T13:41:22.135957Z","iopub.status.idle":"2024-10-16T13:41:22.148683Z","shell.execute_reply.started":"2024-10-16T13:41:22.135916Z","shell.execute_reply":"2024-10-16T13:41:22.147512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\ndataset = dataset_generator(data_path + '/text2ImgData.pkl', BATCH_SIZE, training_data_generator)","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:41:23.450602Z","iopub.execute_input":"2024-10-16T13:41:23.451481Z","iopub.status.idle":"2024-10-16T13:41:27.933003Z","shell.execute_reply.started":"2024-10-16T13:41:23.451432Z","shell.execute_reply":"2024-10-16T13:41:27.931858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers\n\nclass TextEncoder(tf.keras.Model):\n    \"\"\"\n    Encode text (a caption) into hidden representation\n    input: text, which is a list of ids\n    output: embedding, or hidden representation of input text in dimension of RNN_HIDDEN_SIZE\n    \"\"\"\n    def __init__(self, hparas):\n        super(TextEncoder, self).__init__()\n        self.hparas = hparas\n        self.batch_size = self.hparas['BATCH_SIZE']\n        \n        # embedding with tensorflow API\n        self.embedding = layers.Embedding(self.hparas['VOCAB_SIZE'], self.hparas['EMBED_DIM'])\n        # RNN, here we use GRU cell, another common RNN cell similar to LSTM\n        self.gru = layers.GRU(self.hparas['RNN_HIDDEN_SIZE'],\n                              return_sequences=True,\n                              return_state=True,\n                              recurrent_initializer='glorot_uniform')\n    \n    def call(self, text, hidden):\n        text = self.embedding(text)\n        output, state = self.gru(text, initial_state=hidden)\n        return output[:, -1, :], state\n    \n    def initialize_hidden_state(self):\n        return tf.zeros((self.hparas['BATCH_SIZE'], self.hparas['RNN_HIDDEN_SIZE']))\n\nclass Generator(tf.keras.Model):\n    \"\"\"\n    Generate fake image based on given text(hidden representation) and noise z\n    input: text and noise\n    output: fake image with size 64*64*3\n    \"\"\"\n    def __init__(self, hparas):\n        super(Generator, self).__init__()\n        self.hparas = hparas\n        self.flatten = layers.Flatten()\n        self.d1 = layers.Dense(self.hparas['DENSE_DIM'])\n        self.d2 = layers.Dense(64*64*3)\n        \n    def call(self, text, noise_z):\n        text = self.flatten(text)\n        text = self.d1(text)\n        text = tf.nn.leaky_relu(text)\n        \n        # concatenate input text and random noise\n        text_concat = tf.concat([noise_z, text], axis=1)\n        text_concat = self.d2(text_concat)\n        \n        logits = tf.reshape(text_concat, [-1, 64, 64, 3])\n        output = tf.nn.tanh(logits)\n        \n        return logits, output\n\nclass Discriminator(tf.keras.Model):\n    \"\"\"\n    Differentiate the real and fake image\n    input: image and corresponding text\n    output: labels, the real image should be 1, while the fake should be 0\n    \"\"\"\n    def __init__(self, hparas):\n        super(Discriminator, self).__init__()\n        self.hparas = hparas\n        self.flatten = layers.Flatten()\n        self.d_text = layers.Dense(self.hparas['DENSE_DIM'])\n        self.d_img = layers.Dense(self.hparas['DENSE_DIM'])\n        self.d = layers.Dense(1)\n    \n    def call(self, img, text):\n        text = self.flatten(text)\n        text = self.d_text(text)\n        text = tf.nn.leaky_relu(text)\n        \n        img = self.flatten(img)\n        img = self.d_img(img)\n        img = tf.nn.leaky_relu(img)\n        \n        # concatenate image with paired text\n        img_text = tf.concat([text, img], axis=1)\n        \n        logits = self.d(img_text)\n        output = tf.nn.sigmoid(logits)\n        \n        return logits, output\n\n# Define hyperparameters\nhparas = {\n    'MAX_SEQ_LENGTH': 20,\n    'EMBED_DIM': 256,\n    'VOCAB_SIZE': len(word2Id_dict),\n    'RNN_HIDDEN_SIZE': 128,\n    'Z_DIM': 512,\n    'DENSE_DIM': 128,\n    'IMAGE_SIZE': [64, 64, 3],\n    'BATCH_SIZE': 64,\n    'LR': 1e-4,\n    'LR_DECAY': 0.5,\n    'BETA_1': 0.5,\n    'N_EPOCH': 600,\n    'N_SAMPLE': num_training_sample,\n    'CHECKPOINTS_DIR': './checkpoints/demo',\n    'PRINT_FREQ': 1\n}\n\n# Instantiate models\ntext_encoder = TextEncoder(hparas)\ngenerator = Generator(hparas)\ndiscriminator = Discriminator(hparas)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:47:04.231674Z","iopub.execute_input":"2024-10-16T13:47:04.232163Z","iopub.status.idle":"2024-10-16T13:47:04.272753Z","shell.execute_reply.started":"2024-10-16T13:47:04.232109Z","shell.execute_reply":"2024-10-16T13:47:04.271644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:48:42.161016Z","iopub.execute_input":"2024-10-16T13:48:42.161501Z","iopub.status.idle":"2024-10-16T13:48:42.166947Z","shell.execute_reply.started":"2024-10-16T13:48:42.161458Z","shell.execute_reply":"2024-10-16T13:48:42.165808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def discriminator_loss(real_logits, fake_logits):\n    # output value of real image should be 1\n    real_loss = cross_entropy(tf.ones_like(real_logits), real_logits)\n    # output value of fake image should be 0\n    fake_loss = cross_entropy(tf.zeros_like(fake_logits), fake_logits)\n    total_loss = real_loss + fake_loss\n    return total_loss\n\ndef generator_loss(fake_output):\n    # output value of fake image should be 0\n    return cross_entropy(tf.ones_like(fake_output), fake_output)","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:50:11.160828Z","iopub.execute_input":"2024-10-16T13:50:11.161279Z","iopub.status.idle":"2024-10-16T13:50:11.167651Z","shell.execute_reply.started":"2024-10-16T13:50:11.161238Z","shell.execute_reply":"2024-10-16T13:50:11.166518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we use seperated optimizers for training generator and discriminator\ngenerator_optimizer = tf.keras.optimizers.Adam(hparas['LR'])\ndiscriminator_optimizer = tf.keras.optimizers.Adam(hparas['LR'])\n# one benefit of tf.train.Checkpoint() API is we can save everything seperately\ncheckpoint_dir = hparas['CHECKPOINTS_DIR']\ncheckpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\ncheckpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n                                 discriminator_optimizer=discriminator_optimizer,\n                                 text_encoder=text_encoder,\n                                 generator=generator,\n                                 discriminator=discriminator)\n@tf.function\ndef train_step(real_image, caption, hidden):\n    # random noise for generator\n    noise = tf.random.normal(shape=[hparas['BATCH_SIZE'], hparas['Z_DIM']], mean=0.0, stddev=1.0)\n    \n    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n        text_embed, hidden = text_encoder(caption, hidden)\n        _, fake_image = generator(text_embed, noise)\n        real_logits, real_output = discriminator(real_image, text_embed)\n        fake_logits, fake_output = discriminator(fake_image, text_embed)\n\n        g_loss = generator_loss(fake_logits)\n        d_loss = discriminator_loss(real_logits, fake_logits)\n\n    grad_g = gen_tape.gradient(g_loss, generator.trainable_variables)\n    grad_d = disc_tape.gradient(d_loss, discriminator.trainable_variables)\n\n    generator_optimizer.apply_gradients(zip(grad_g, generator.trainable_variables))\n    discriminator_optimizer.apply_gradients(zip(grad_d, discriminator.trainable_variables))\n    \n    return g_loss, d_loss","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:50:27.805676Z","iopub.execute_input":"2024-10-16T13:50:27.806162Z","iopub.status.idle":"2024-10-16T13:50:27.831223Z","shell.execute_reply.started":"2024-10-16T13:50:27.806103Z","shell.execute_reply":"2024-10-16T13:50:27.829539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef test_step(caption, noise, hidden):\n    text_embed, hidden = text_encoder(caption, hidden)\n    _, fake_image = generator(text_embed, noise)\n    return fake_image","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:52:46.926619Z","iopub.execute_input":"2024-10-16T13:52:46.927088Z","iopub.status.idle":"2024-10-16T13:52:46.933772Z","shell.execute_reply.started":"2024-10-16T13:52:46.927043Z","shell.execute_reply":"2024-10-16T13:52:46.932489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge(images, size):\n    h, w = images.shape[1], images.shape[2]\n    img = np.zeros((h * size[0], w * size[1], 3))\n    for idx, image in enumerate(images):\n        i = idx % size[1]\n        j = idx // size[1]\n        img[j*h:j*h+h, i*w:i*w+w, :] = image\n    return img\n\ndef imsave(images, size, path):\n    # getting the pixel values between [0, 1] to save it\n    return plt.imsave(path, merge(images, size)*0.5 + 0.5)\n\ndef save_images(images, size, image_path):\n    return imsave(images, size, image_path)\ndef sample_generator(caption, batch_size):\n    caption = np.asarray(caption)\n    caption = caption.astype(np.int32)\n    dataset = tf.data.Dataset.from_tensor_slices(caption)\n    dataset = dataset.batch(batch_size)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:54:54.970874Z","iopub.execute_input":"2024-10-16T13:54:54.971679Z","iopub.status.idle":"2024-10-16T13:54:54.981731Z","shell.execute_reply.started":"2024-10-16T13:54:54.971635Z","shell.execute_reply":"2024-10-16T13:54:54.980642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ni = int(np.ceil(np.sqrt(hparas['BATCH_SIZE'])))\nsample_size = hparas['BATCH_SIZE']\nsample_seed = np.random.normal(loc=0.0, scale=1.0, size=(sample_size, hparas['Z_DIM'])).astype(np.float32)\nsample_sentence = [\"the flower shown has yellow anther red pistil and bright red petals.\"] * int(sample_size/ni) + \\\n                  [\"this flower has petals that are yellow, white and purple and has dark lines\"] * int(sample_size/ni) + \\\n                  [\"the petals on this flower are white with a yellow center\"] * int(sample_size/ni) + \\\n                  [\"this flower has a lot of small round pink petals.\"] * int(sample_size/ni) + \\\n                  [\"this flower is orange in color, and has petals that are ruffled and rounded.\"] * int(sample_size/ni) + \\\n                  [\"the flower has yellow petals and the center of it is brown.\"] * int(sample_size/ni) + \\\n                  [\"this flower has petals that are blue and white.\"] * int(sample_size/ni) +\\\n                  [\"these white flowers have petals that start off white in color and end in a white towards the tips.\"] * int(sample_size/ni)\n\nfor i, sent in enumerate(sample_sentence):\n    sample_sentence[i] = sent2IdList(sent)\nsample_sentence = sample_generator(sample_sentence, hparas['BATCH_SIZE'])","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:54:56.449801Z","iopub.execute_input":"2024-10-16T13:54:56.450267Z","iopub.status.idle":"2024-10-16T13:54:56.470625Z","shell.execute_reply.started":"2024-10-16T13:54:56.450226Z","shell.execute_reply":"2024-10-16T13:54:56.469512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('samples/demo'):\n    os.makedirs('samples/demo')","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:55:06.365243Z","iopub.execute_input":"2024-10-16T13:55:06.365711Z","iopub.status.idle":"2024-10-16T13:55:06.371249Z","shell.execute_reply.started":"2024-10-16T13:55:06.365669Z","shell.execute_reply":"2024-10-16T13:55:06.370074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(dataset, epochs):\n    # hidden state of RNN\n    hidden = text_encoder.initialize_hidden_state()\n    steps_per_epoch = int(hparas['N_SAMPLE']/hparas['BATCH_SIZE'])\n    \n    for epoch in range(hparas['N_EPOCH']):\n        g_total_loss = 0\n        d_total_loss = 0\n        start = time.time()\n        \n        for image, caption in dataset:\n            g_loss, d_loss = train_step(image, caption, hidden)\n            g_total_loss += g_loss\n            d_total_loss += d_loss\n            \n        time_tuple = time.localtime()\n        time_string = time.strftime(\"%m/%d/%Y, %H:%M:%S\", time_tuple)\n            \n        print(\"Epoch {}, gen_loss: {:.4f}, disc_loss: {:.4f}\".format(epoch+1,\n                                                                     g_total_loss/steps_per_epoch,\n                                                                     d_total_loss/steps_per_epoch))\n        print('Time for epoch {} is {:.4f} sec'.format(epoch+1, time.time()-start))\n        \n        # save the model\n        if (epoch + 1) % 50 == 0:\n            checkpoint.save(file_prefix = checkpoint_prefix)\n        \n        # visualization\n        if (epoch + 1) % hparas['PRINT_FREQ'] == 0:\n            for caption in sample_sentence:\n                fake_image = test_step(caption, sample_seed, hidden)\n            save_images(fake_image, [ni, ni], 'samples/demo/train_{:02d}.jpg'.format(epoch))","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:55:21.656619Z","iopub.execute_input":"2024-10-16T13:55:21.657458Z","iopub.status.idle":"2024-10-16T13:55:21.668197Z","shell.execute_reply.started":"2024-10-16T13:55:21.657408Z","shell.execute_reply":"2024-10-16T13:55:21.666939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(dataset, hparas['N_EPOCH'])","metadata":{"execution":{"iopub.status.busy":"2024-10-16T13:55:42.055435Z","iopub.execute_input":"2024-10-16T13:55:42.055867Z","iopub.status.idle":"2024-10-16T13:55:43.687539Z","shell.execute_reply.started":"2024-10-16T13:55:42.05583Z","shell.execute_reply":"2024-10-16T13:55:43.686019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}