{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport torch\nfrom torch.jit import script, trace\nimport torch.nn as nn\nfrom torch import optim\nimport torch.nn.functional as F\nimport csv\nimport random\nimport re\nimport os\nimport unicodedata\nimport codecs\nfrom io import open\nimport itertools\nimport math\nimport json\nimport random\n\nUSE_CUDA = torch.cuda.is_available()\ndevice = torch.device(\"cuda\" if USE_CUDA else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:22:33.330346Z","iopub.execute_input":"2022-12-22T12:22:33.331134Z","iopub.status.idle":"2022-12-22T12:22:35.594982Z","shell.execute_reply.started":"2022-12-22T12:22:33.331015Z","shell.execute_reply":"2022-12-22T12:22:35.593477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:22:35.597342Z","iopub.execute_input":"2022-12-22T12:22:35.598436Z","iopub.status.idle":"2022-12-22T12:22:35.615321Z","shell.execute_reply.started":"2022-12-22T12:22:35.598384Z","shell.execute_reply":"2022-12-22T12:22:35.613921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Default word tokens\nPAD_token = 0  # Used for padding short sentences\nSOS_token = 1  # Start-of-sentence token\nEOS_token = 2  # End-of-sentence token\n\nclass Voc:\n    def __init__(self, name):\n        self.name = name\n        self.trimmed = False\n        self.word2index = {}\n        self.word2count = {}\n        self.index2word = {PAD_token: \"PAD\", SOS_token: \"SOS\", EOS_token: \"EOS\"}\n        self.num_words = 3  # Count SOS, EOS, PAD\n\n    def addSentence(self, sentence):\n        for word in sentence.split(' '):\n            self.addWord(word)\n\n    def addWord(self, word):\n        if word not in self.word2index:\n            self.word2index[word] = self.num_words\n            self.word2count[word] = 1\n            self.index2word[self.num_words] = word\n            self.num_words += 1\n        else:\n            self.word2count[word] += 1\n\n    # Remove words below a certain count threshold\n    def trim(self, min_count):\n        if self.trimmed:\n            return\n        self.trimmed = True\n\n        keep_words = []\n\n        for k, v in self.word2count.items():\n            if v >= min_count:\n                keep_words.append(k)\n\n        print('keep_words {} / {} = {:.4f}'.format(\n            len(keep_words), len(self.word2index), len(keep_words) / len(self.word2index)\n        ))\n\n        # Reinitialize dictionaries\n        self.word2index = {}\n        self.word2count = {}\n        self.index2word = {PAD_token: \"PAD\", SOS_token: \"SOS\", EOS_token: \"EOS\"}\n        self.num_words = 3 # Count default tokens\n\n        for word in keep_words:\n            self.addWord(word)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:22:35.621651Z","iopub.execute_input":"2022-12-22T12:22:35.622724Z","iopub.status.idle":"2022-12-22T12:22:35.644266Z","shell.execute_reply.started":"2022-12-22T12:22:35.622658Z","shell.execute_reply":"2022-12-22T12:22:35.642923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 10  # Maximum sentence length to consider\n\n\ndef normalizeString(s):\n    s = s.lower().strip()\n    return s\n\n# Read query/response pairs and return a voc object\ndef readVocs(datafile, corpus_name):\n    print(\"Reading lines...\")\n    # Read the file and split into lines\n    lines = open(datafile, encoding='utf-8').\\\n        read().strip().split('\\n')\n    # Split every line into pairs and normalize\n    pairs = [[normalizeString(s) for s in l.split('\\t')] for l in lines]\n    voc = Voc(corpus_name)\n    return voc, pairs\n\n# Returns True if both sentences in a pair 'p' are under the MAX_LENGTH threshold\ndef filterPair(p):\n    # Input sequences need to preserve the last word for EOS token\n    return len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH\n\n# Filter pairs using filterPair condition\ndef filterPairs(pairs):\n    return [pair for pair in pairs if filterPair(pair)]\n\n# Using the functions defined above, return a populated voc object and pairs list\ndef loadPrepareData(corpus_name, datafile, save_dir):\n    print(\"Start preparing training data ...\")\n    voc, pairs = readVocs(datafile, corpus_name)\n    print(\"Read {!s} sentence pairs\".format(len(pairs)))\n    pairs = filterPairs(pairs)\n    print(\"Trimmed to {!s} sentence pairs\".format(len(pairs)))\n    print(\"Counting words...\")\n    for pair in pairs:\n        voc.addSentence(pair[0])\n        voc.addSentence(pair[1])\n    print(\"Counted words:\", voc.num_words)\n    return voc, pairs\n\n\n# Load/Assemble voc and pairs\nsave_dir = os.path.join(\"data\", \"save\")\ncorpus_name = 'chit_chat_corpus'\ndatafile = '/kaggle/input/chit-chat-encoders/train.txt'\nvoc, pairs = loadPrepareData(corpus_name, datafile, save_dir)\n# Print some pairs to validate\nprint(\"\\npairs:\")\nfor pair in pairs[:10]:\n    print(pair)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:22:35.648124Z","iopub.execute_input":"2022-12-22T12:22:35.649410Z","iopub.status.idle":"2022-12-22T12:24:30.665608Z","shell.execute_reply.started":"2022-12-22T12:22:35.649337Z","shell.execute_reply":"2022-12-22T12:24:30.663911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!!!!\npairs = pairs[:100]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:30.667092Z","iopub.execute_input":"2022-12-22T12:24:30.667451Z","iopub.status.idle":"2022-12-22T12:24:31.353506Z","shell.execute_reply.started":"2022-12-22T12:24:30.667418Z","shell.execute_reply":"2022-12-22T12:24:31.352661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MIN_COUNT = 3    # Minimum word count threshold for trimming\n\ndef trimRareWords(voc, pairs, MIN_COUNT):\n    # Trim words used under the MIN_COUNT from the voc\n    voc.trim(MIN_COUNT)\n    # Filter out pairs with trimmed words\n    keep_pairs = []\n    for pair in pairs:\n        input_sentence = pair[0]\n        output_sentence = pair[1]\n        keep_input = True\n        keep_output = True\n        # Check input sentence\n        for word in input_sentence.split(' '):\n            if word not in voc.word2index:\n                keep_input = False\n                break\n        # Check output sentence\n        for word in output_sentence.split(' '):\n            if word not in voc.word2index:\n                keep_output = False\n                break\n\n        # Only keep pairs that do not contain trimmed word(s) in their input or output sentence\n        if keep_input and keep_output:\n            keep_pairs.append(pair)\n\n    print(\"Trimmed from {} pairs to {}, {:.4f} of total\".format(len(pairs), len(keep_pairs), len(keep_pairs) / len(pairs)))\n    return keep_pairs\n\n\n# Trim voc and pairs\npairs = trimRareWords(voc, pairs, MIN_COUNT)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:31.354743Z","iopub.execute_input":"2022-12-22T12:24:31.355258Z","iopub.status.idle":"2022-12-22T12:24:33.775732Z","shell.execute_reply.started":"2022-12-22T12:24:31.355226Z","shell.execute_reply":"2022-12-22T12:24:33.774440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def indexesFromSentence(voc, sentence):\n    return [voc.word2index[word] for word in sentence.split(' ')] + [EOS_token]\n\n\ndef zeroPadding(l, fillvalue=PAD_token):\n    return list(itertools.zip_longest(*l, fillvalue=fillvalue))\n\ndef binaryMatrix(l, value=PAD_token):\n    m = []\n    for i, seq in enumerate(l):\n        m.append([])\n        for token in seq:\n            if token == PAD_token:\n                m[i].append(0)\n            else:\n                m[i].append(1)\n    return m\n\n# Returns padded input sequence tensor and lengths\ndef inputVar(l, voc):\n    indexes_batch = [indexesFromSentence(voc, sentence) for sentence in l]\n    lengths = torch.tensor([len(indexes) for indexes in indexes_batch])\n    padList = zeroPadding(indexes_batch)\n    padVar = torch.LongTensor(padList)\n    return padVar, lengths\n\n# Returns padded target sequence tensor, padding mask, and max target length\ndef outputVar(l, voc):\n    indexes_batch = [indexesFromSentence(voc, sentence) for sentence in l]\n    max_target_len = max([len(indexes) for indexes in indexes_batch])\n    padList = zeroPadding(indexes_batch)\n    mask = binaryMatrix(padList)\n    mask = torch.BoolTensor(mask)\n    padVar = torch.LongTensor(padList)\n    return padVar, mask, max_target_len\n\n# Returns all items for a given batch of pairs\ndef batch2TrainData(voc, pair_batch):\n    pair_batch.sort(key=lambda x: len(x[0].split(\" \")), reverse=True)\n    input_batch, output_batch = [], []\n    for pair in pair_batch:\n        input_batch.append(pair[0])\n        output_batch.append(pair[1])\n    inp, lengths = inputVar(input_batch, voc)\n    output, mask, max_target_len = outputVar(output_batch, voc)\n    return inp, lengths, output, mask, max_target_len\n\n\n# Example for validation\nsmall_batch_size = 4\nbatches = batch2TrainData(voc, [random.choice(pairs) for _ in range(small_batch_size)])\ninput_variable, lengths, target_variable, mask, max_target_len = batches\n\nprint(\"input_variable:\", input_variable)\nprint(\"lengths:\", lengths)\nprint(\"target_variable:\", target_variable)\nprint(\"mask:\", mask)\nprint(\"max_target_len:\", max_target_len)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.777560Z","iopub.execute_input":"2022-12-22T12:24:33.777920Z","iopub.status.idle":"2022-12-22T12:24:33.797822Z","shell.execute_reply.started":"2022-12-22T12:24:33.777889Z","shell.execute_reply":"2022-12-22T12:24:33.796189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderRNN(nn.Module):\n    def __init__(self, hidden_size, embedding, n_layers=1, dropout=0):\n        super(EncoderRNN, self).__init__()\n        self.n_layers = n_layers\n        self.hidden_size = hidden_size\n        self.embedding = embedding\n\n        # Initialize GRU; the input_size and hidden_size params are both set to 'hidden_size'\n        #   because our input size is a word embedding with number of features == hidden_size\n        self.gru = nn.GRU(hidden_size, hidden_size, n_layers,\n                          dropout=(0 if n_layers == 1 else dropout), bidirectional=True)\n\n    def forward(self, input_seq, input_lengths, hidden=None):\n        # Convert word indexes to embeddings\n        embedded = self.embedding(input_seq)\n        # Pack padded batch of sequences for RNN module\n        packed = nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)\n        # Forward pass through GRU\n        outputs, hidden = self.gru(packed, hidden)\n        # Unpack padding\n        outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs)\n        # Sum bidirectional GRU outputs\n        outputs = outputs[:, :, :self.hidden_size] + outputs[:, : ,self.hidden_size:]\n        # Return output and final hidden state\n        return outputs, hidden","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.800318Z","iopub.execute_input":"2022-12-22T12:24:33.800870Z","iopub.status.idle":"2022-12-22T12:24:33.812897Z","shell.execute_reply.started":"2022-12-22T12:24:33.800822Z","shell.execute_reply":"2022-12-22T12:24:33.811409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Luong attention layer\nclass Attn(nn.Module):\n    def __init__(self, method, hidden_size):\n        super(Attn, self).__init__()\n        self.method = method\n        if self.method not in ['dot', 'general', 'concat']:\n            raise ValueError(self.method, \"is not an appropriate attention method.\")\n        self.hidden_size = hidden_size\n        if self.method == 'general':\n            self.attn = nn.Linear(self.hidden_size, hidden_size)\n        elif self.method == 'concat':\n            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)\n            self.v = nn.Parameter(torch.FloatTensor(hidden_size))\n\n    def dot_score(self, hidden, encoder_output):\n        return torch.sum(hidden * encoder_output, dim=2)\n\n    def general_score(self, hidden, encoder_output):\n        energy = self.attn(encoder_output)\n        return torch.sum(hidden * energy, dim=2)\n\n    def concat_score(self, hidden, encoder_output):\n        energy = self.attn(torch.cat((hidden.expand(encoder_output.size(0), -1, -1), encoder_output), 2)).tanh()\n        return torch.sum(self.v * energy, dim=2)\n\n    def forward(self, hidden, encoder_outputs):\n        # Calculate the attention weights (energies) based on the given method\n        if self.method == 'general':\n            attn_energies = self.general_score(hidden, encoder_outputs)\n        elif self.method == 'concat':\n            attn_energies = self.concat_score(hidden, encoder_outputs)\n        elif self.method == 'dot':\n            attn_energies = self.dot_score(hidden, encoder_outputs)\n\n        # Transpose max_length and batch_size dimensions\n        attn_energies = attn_energies.t()\n\n        # Return the softmax normalized probability scores (with added dimension)\n        return F.softmax(attn_energies, dim=1).unsqueeze(1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.814707Z","iopub.execute_input":"2022-12-22T12:24:33.815039Z","iopub.status.idle":"2022-12-22T12:24:33.832596Z","shell.execute_reply.started":"2022-12-22T12:24:33.815010Z","shell.execute_reply":"2022-12-22T12:24:33.831437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LuongAttnDecoderRNN(nn.Module):\n    def __init__(self, attn_model, embedding, hidden_size, output_size, n_layers=1, dropout=0.1):\n        super(LuongAttnDecoderRNN, self).__init__()\n\n        # Keep for reference\n        self.attn_model = attn_model\n        self.hidden_size = hidden_size\n        self.output_size = output_size\n        self.n_layers = n_layers\n        self.dropout = dropout\n\n        # Define layers\n        self.embedding = embedding\n        self.embedding_dropout = nn.Dropout(dropout)\n        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=(0 if n_layers == 1 else dropout))\n        self.concat = nn.Linear(hidden_size * 2, hidden_size)\n        self.out = nn.Linear(hidden_size, output_size)\n\n        self.attn = Attn(attn_model, hidden_size)\n\n    def forward(self, input_step, last_hidden, encoder_outputs):\n        # Note: we run this one step (word) at a time\n        # Get embedding of current input word\n        embedded = self.embedding(input_step)\n        embedded = self.embedding_dropout(embedded)\n        # Forward through unidirectional GRU\n        rnn_output, hidden = self.gru(embedded, last_hidden)\n        # Calculate attention weights from the current GRU output\n        attn_weights = self.attn(rnn_output, encoder_outputs)\n        # Multiply attention weights to encoder outputs to get new \"weighted sum\" context vector\n        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))\n        # Concatenate weighted context vector and GRU output using Luong eq. 5\n        rnn_output = rnn_output.squeeze(0)\n        context = context.squeeze(1)\n        concat_input = torch.cat((rnn_output, context), 1)\n        concat_output = torch.tanh(self.concat(concat_input))\n        # Predict next word using Luong eq. 6\n        output = self.out(concat_output)\n        output = F.softmax(output, dim=1)\n        # Return output and final hidden state\n        return output, hidden","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.837797Z","iopub.execute_input":"2022-12-22T12:24:33.838216Z","iopub.status.idle":"2022-12-22T12:24:33.851962Z","shell.execute_reply.started":"2022-12-22T12:24:33.838182Z","shell.execute_reply":"2022-12-22T12:24:33.850883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def maskNLLLoss(inp, target, mask):\n    nTotal = mask.sum()\n    crossEntropy = -torch.log(torch.gather(inp, 1, target.view(-1, 1)).squeeze(1))\n    loss = crossEntropy.masked_select(mask).mean()\n    loss = loss.to(device)\n    return loss, nTotal.item()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.853530Z","iopub.execute_input":"2022-12-22T12:24:33.853897Z","iopub.status.idle":"2022-12-22T12:24:33.869382Z","shell.execute_reply.started":"2022-12-22T12:24:33.853865Z","shell.execute_reply":"2022-12-22T12:24:33.868401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(input_variable, lengths, target_variable, mask, max_target_len, encoder, decoder, embedding,\n          encoder_optimizer, decoder_optimizer, batch_size, clip, max_length=MAX_LENGTH):\n\n    # Zero gradients\n    encoder_optimizer.zero_grad()\n    decoder_optimizer.zero_grad()\n\n    # Set device options\n    input_variable = input_variable.to(device)\n    target_variable = target_variable.to(device)\n    mask = mask.to(device)\n    # Lengths for rnn packing should always be on the cpu\n    lengths = lengths.to(\"cpu\")\n\n    # Initialize variables\n    loss = 0\n    print_losses = []\n    n_totals = 0\n\n    # Forward pass through encoder\n    encoder_outputs, encoder_hidden = encoder(input_variable, lengths)\n\n    # Create initial decoder input (start with SOS tokens for each sentence)\n    decoder_input = torch.LongTensor([[SOS_token for _ in range(batch_size)]])\n    decoder_input = decoder_input.to(device)\n\n    # Set initial decoder hidden state to the encoder's final hidden state\n    decoder_hidden = encoder_hidden[:decoder.n_layers]\n\n    # Determine if we are using teacher forcing this iteration\n    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n\n    # Forward batch of sequences through decoder one time step at a time\n    if use_teacher_forcing:\n        for t in range(max_target_len):\n            decoder_output, decoder_hidden = decoder(\n                decoder_input, decoder_hidden, encoder_outputs\n            )\n            # Teacher forcing: next input is current target\n            decoder_input = target_variable[t].view(1, -1)\n            # Calculate and accumulate loss\n            mask_loss, nTotal = maskNLLLoss(decoder_output, target_variable[t], mask[t])\n            loss += mask_loss\n            print_losses.append(mask_loss.item() * nTotal)\n            n_totals += nTotal\n    else:\n        for t in range(max_target_len):\n            decoder_output, decoder_hidden = decoder(\n                decoder_input, decoder_hidden, encoder_outputs\n            )\n            # No teacher forcing: next input is decoder's own current output\n            _, topi = decoder_output.topk(1)\n            decoder_input = torch.LongTensor([[topi[i][0] for i in range(batch_size)]])\n            decoder_input = decoder_input.to(device)\n            # Calculate and accumulate loss\n            mask_loss, nTotal = maskNLLLoss(decoder_output, target_variable[t], mask[t])\n            loss += mask_loss\n            print_losses.append(mask_loss.item() * nTotal)\n            n_totals += nTotal\n\n    # Perform backpropatation\n    loss.backward()\n\n    # Clip gradients: gradients are modified in place\n    _ = nn.utils.clip_grad_norm_(encoder.parameters(), clip)\n    _ = nn.utils.clip_grad_norm_(decoder.parameters(), clip)\n\n    # Adjust model weights\n    encoder_optimizer.step()\n    decoder_optimizer.step()\n\n    return sum(print_losses) / n_totals","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.871173Z","iopub.execute_input":"2022-12-22T12:24:33.871544Z","iopub.status.idle":"2022-12-22T12:24:33.888265Z","shell.execute_reply.started":"2022-12-22T12:24:33.871512Z","shell.execute_reply":"2022-12-22T12:24:33.887228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer, embedding, encoder_n_layers, decoder_n_layers, save_dir, n_iteration, batch_size, print_every, save_every, clip, corpus_name, loadFilename):\n\n    # Load batches for each iteration\n    training_batches = [batch2TrainData(voc, [random.choice(pairs) for _ in range(batch_size)])\n                      for _ in range(n_iteration)]\n\n    # Initializations\n    print('Initializing ...')\n    start_iteration = 1\n    print_loss = 0\n    if loadFilename:\n        start_iteration = checkpoint['iteration'] + 1\n\n    # Training loop\n    print(\"Training...\")\n    for iteration in range(start_iteration, n_iteration + 1):\n        training_batch = training_batches[iteration - 1]\n        # Extract fields from batch\n        input_variable, lengths, target_variable, mask, max_target_len = training_batch\n\n        # Run a training iteration with batch\n        loss = train(input_variable, lengths, target_variable, mask, max_target_len, encoder,\n                     decoder, embedding, encoder_optimizer, decoder_optimizer, batch_size, clip)\n        print_loss += loss\n\n        # Print progress\n        if iteration % print_every == 0:\n            print_loss_avg = print_loss / print_every\n            print(\"Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}\".format(iteration, iteration / n_iteration * 100, print_loss_avg))\n            print_loss = 0\n\n        # Save checkpoint\n        if (iteration % save_every == 0):\n            directory = os.path.join(save_dir, model_name, corpus_name, '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size))\n            if not os.path.exists(directory):\n                os.makedirs(directory)\n            torch.save({\n                'iteration': iteration,\n                'en': encoder.state_dict(),\n                'de': decoder.state_dict(),\n                'en_opt': encoder_optimizer.state_dict(),\n                'de_opt': decoder_optimizer.state_dict(),\n                'loss': loss,\n                'voc_dict': voc.__dict__,\n                'embedding': embedding.state_dict()\n            }, os.path.join(directory, '{}_{}.tar'.format(iteration, 'checkpoint')))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.889442Z","iopub.execute_input":"2022-12-22T12:24:33.889805Z","iopub.status.idle":"2022-12-22T12:24:33.906376Z","shell.execute_reply.started":"2022-12-22T12:24:33.889773Z","shell.execute_reply":"2022-12-22T12:24:33.905061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GreedySearchDecoder(nn.Module):\n    def __init__(self, encoder, decoder):\n        super(GreedySearchDecoder, self).__init__()\n        self.encoder = encoder\n        self.decoder = decoder\n\n    def forward(self, input_seq, input_length, max_length):\n        # Forward input through encoder model\n        encoder_outputs, encoder_hidden = self.encoder(input_seq, input_length)\n        # Prepare encoder's final hidden layer to be first hidden input to the decoder\n        decoder_hidden = encoder_hidden[:decoder.n_layers]\n        # Initialize decoder input with SOS_token\n        decoder_input = torch.ones(1, 1, device=device, dtype=torch.long) * SOS_token\n        # Initialize tensors to append decoded words to\n        all_tokens = torch.zeros([0], device=device, dtype=torch.long)\n        all_scores = torch.zeros([0], device=device)\n        # Iteratively decode one word token at a time\n        for _ in range(max_length):\n            # Forward pass through decoder\n            decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden, encoder_outputs)\n            # Obtain most likely word token and its softmax score\n            decoder_scores, decoder_input = torch.max(decoder_output, dim=1)\n            # Record token and score\n            all_tokens = torch.cat((all_tokens, decoder_input), dim=0)\n            all_scores = torch.cat((all_scores, decoder_scores), dim=0)\n            # Prepare current token to be next decoder input (add a dimension)\n            decoder_input = torch.unsqueeze(decoder_input, 0)\n        # Return collections of word tokens and scores\n        return all_tokens, all_scores","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.907528Z","iopub.execute_input":"2022-12-22T12:24:33.908410Z","iopub.status.idle":"2022-12-22T12:24:33.923600Z","shell.execute_reply.started":"2022-12-22T12:24:33.908352Z","shell.execute_reply":"2022-12-22T12:24:33.922669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(encoder, decoder, searcher, voc, sentence, max_length=MAX_LENGTH):\n    ### Format input sentence as a batch\n    # words -> indexes\n    indexes_batch = [indexesFromSentence(voc, sentence)]\n    # Create lengths tensor\n    lengths = torch.tensor([len(indexes) for indexes in indexes_batch])\n    # Transpose dimensions of batch to match models' expectations\n    input_batch = torch.LongTensor(indexes_batch).transpose(0, 1)\n    # Use appropriate device\n    input_batch = input_batch.to(device)\n    lengths = lengths.to(\"cpu\")\n    # Decode sentence with searcher\n    tokens, scores = searcher(input_batch, lengths, max_length)\n    # indexes -> words\n    decoded_words = [voc.index2word[token.item()] for token in tokens]\n    return decoded_words\n\n\ndef evaluateInput(encoder, decoder, searcher, voc):\n    input_sentence = ''\n    while(1):\n        try:\n            # Get input sentence\n            input_sentence = input('> ')\n            # Check if it is quit case\n            if input_sentence == 'q' or input_sentence == 'quit': break\n            # Normalize sentence\n            input_sentence = normalizeString(input_sentence)\n            # Evaluate sentence\n            output_words = evaluate(encoder, decoder, searcher, voc, input_sentence)\n            # Format and print response sentence\n            output_words[:] = [x for x in output_words if not (x == 'EOS' or x == 'PAD')]\n            print('Bot:', ' '.join(output_words))\n\n        except KeyError:\n            print(\"Error: Encountered unknown word.\")","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.924988Z","iopub.execute_input":"2022-12-22T12:24:33.925537Z","iopub.status.idle":"2022-12-22T12:24:33.940787Z","shell.execute_reply.started":"2022-12-22T12:24:33.925503Z","shell.execute_reply":"2022-12-22T12:24:33.939688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configure models\nmodel_name = 'cb_model'\nattn_model = 'dot'\n#attn_model = 'general'\n#attn_model = 'concat'\nhidden_size = 500\nencoder_n_layers = 2\ndecoder_n_layers = 2\ndropout = 0.1\nbatch_size = 4\n\n# Set checkpoint to load from; set to None if starting from scratch\nloadFilename = None\ncheckpoint_iter = 4000\n#loadFilename = os.path.join(save_dir, model_name, corpus_name,\n#                            '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size),\n#                            '{}_checkpoint.tar'.format(checkpoint_iter))\n\n\n# Load model if a loadFilename is provided\nif loadFilename:\n    # If loading on same machine the model was trained on\n    checkpoint = torch.load(loadFilename)\n    # If loading a model trained on GPU to CPU\n    #checkpoint = torch.load(loadFilename, map_location=torch.device('cpu'))\n    encoder_sd = checkpoint['en']\n    decoder_sd = checkpoint['de']\n    encoder_optimizer_sd = checkpoint['en_opt']\n    decoder_optimizer_sd = checkpoint['de_opt']\n    embedding_sd = checkpoint['embedding']\n    voc.__dict__ = checkpoint['voc_dict']\n\n\nprint('Building encoder and decoder ...')\n# Initialize word embeddings\nembedding = nn.Embedding(voc.num_words, hidden_size)\nif loadFilename:\n    embedding.load_state_dict(embedding_sd)\n# Initialize encoder & decoder models\nencoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout)\ndecoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, voc.num_words, decoder_n_layers, dropout)\nif loadFilename:\n    encoder.load_state_dict(encoder_sd)\n    decoder.load_state_dict(decoder_sd)\n# Use appropriate device\nencoder = encoder.to(device)\ndecoder = decoder.to(device)\nprint('Models built and ready to go!')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:33.942041Z","iopub.execute_input":"2022-12-22T12:24:33.942857Z","iopub.status.idle":"2022-12-22T12:24:42.112767Z","shell.execute_reply.started":"2022-12-22T12:24:33.942822Z","shell.execute_reply":"2022-12-22T12:24:42.111145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configure training/optimization\nclip = 50.0\nteacher_forcing_ratio = 1.0\nlearning_rate = 0.0001\ndecoder_learning_ratio = 5.0\nn_iteration = 4000\nprint_every = 1\nsave_every = 500\n\n# Ensure dropout layers are in train mode\nencoder.train()\ndecoder.train()\n\n# Initialize optimizers\nprint('Building optimizers ...')\nencoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)\ndecoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio)\nif loadFilename:\n    encoder_optimizer.load_state_dict(encoder_optimizer_sd)\n    decoder_optimizer.load_state_dict(decoder_optimizer_sd)\n\n# If you have cuda, configure cuda to call\nfor state in encoder_optimizer.state.values():\n    for k, v in state.items():\n        if isinstance(v, torch.Tensor):\n            state[k] = v.cuda()\n\nfor state in decoder_optimizer.state.values():\n    for k, v in state.items():\n        if isinstance(v, torch.Tensor):\n            state[k] = v.cuda()\n\n# Run training iterations\nprint(\"Starting Training!\")\ntrainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer,\n           embedding, encoder_n_layers, decoder_n_layers, save_dir, n_iteration, batch_size,\n           print_every, save_every, clip, corpus_name, loadFilename)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T12:24:42.114475Z","iopub.execute_input":"2022-12-22T12:24:42.114842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set dropout layers to eval mode\nencoder.eval()\ndecoder.eval()\n\n# Initialize search module\nsearcher = GreedySearchDecoder(encoder, decoder)\n\n# Begin chatting (uncomment and run the following line to begin)\n# evaluateInput(encoder, decoder, searcher, voc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluateInput(encoder, decoder, searcher, voc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}