{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# -*- coding: utf-8 -*-\nimport collections\nimport math\nimport os\nimport numpy as np\nimport random\nfrom six.moves import xrange\nfrom sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nif tf.__version__[0] == '2':\n    # Using TensorFlow 1.x to train our word2vec\n    import tensorflow.compat.v1 as tf\n    tf.disable_v2_behavior()\n%matplotlib inline ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder  = \"/kaggle/input/109-2-uta-dl-app-hw2\"\nfilename   = \"tag_list.txt\"  # Hashtag list\nvocabulary_size = 990","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size     = 128\nembedding_size = 64       # Dimension of the embedding vector.\nskip_window    = 1         # How many words to consider left and right.\nnum_skips      = 2         # How many times to reuse an input ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Random validation set to sample nearest neighbors.\nvalid_size     = 32        # Random set of words to evaluate similarity \nvalid_window   = 200       # Only pick validation samples in the top 200\nvalid_examples = np.random.choice(valid_window, valid_size, replace=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path   = os.path.join(data_folder, filename)\nwith open(file_path, 'r', encoding=\"utf-8\") as f:\n    words = f.read().split()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words[:20]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_count = [['UNK', -1]] \nword_count.extend(collections.Counter(words)\n             .most_common(vocabulary_size - 1)) # -1 is for UNK \nprint (\"%s\" % (word_count[0:10]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create word -> wordID dictionary\ndictionary = dict() \nfor word, _ in word_count:\n    dictionary[word] = len(dictionary)\n\n# Create reverse dictionary (wordID -> word)\nreverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert word into wordID, and count unused words (UNK)\ndata = list()\nunk_count = 0\nfor word in words:\n    if word in dictionary:\n        index = dictionary[word]\n    else:\n        index = 0  # dictionary['UNK']\n        unk_count += 1\n    data.append(index)\nword_count[0][1] = unk_count\n# del words  # Hint to reduce memory.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"Most common words (+UNK) are: %s\" % (word_count[:10]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"Sample data corresponds to\\n__________________\")\nfor i in range(10):\n    print (\"%d->%s\" % (data[i], reverse_dictionary[data[i]]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_index = 0\ndef generate_batch(batch_size, num_skips, skip_window):\n    global data_index\n    assert batch_size % num_skips == 0\n    assert num_skips <= 2 * skip_window\n    batch  = np.ndarray(shape=(batch_size),    dtype=np.int32)\n    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n    span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n    buffer = collections.deque(maxlen=span)\n    for _ in range(span):\n        buffer.append(data[data_index])\n        data_index = (data_index + 1) % len(data)\n    for i in range(batch_size // num_skips): # '//' makes the result an integer, e.g., 7//3 = 2\n        target = skip_window\n        targets_to_avoid = [ skip_window ]\n        for j in range(num_skips):\n            while target in targets_to_avoid:\n                target = random.randint(0, span - 1)\n            targets_to_avoid.append(target)\n            batch[i * num_skips + j] = buffer[skip_window]\n            labels[i * num_skips + j, 0] = buffer[target]\n        buffer.append(data[data_index])\n        data_index = (data_index + 1) % len(data)\n    return batch, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Construct the word2vec model \ntrain_inputs   = tf.placeholder(tf.int32, shape=[batch_size])   \ntrain_labels   = tf.placeholder(tf.int32, shape=[batch_size, 1])\nvalid_dataset  = tf.constant(valid_examples, dtype=tf.int32)\n\n# Look up embeddings for inputs. (vocabulary_size = 50,000)\nwith tf.variable_scope(\"EMBEDDING\"):\n    with tf.device('/cpu:0'):\n        embeddings = tf.Variable(\n            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n        embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n    \n# Construct the variables for the NCE loss\nwith tf.variable_scope(\"NCE_WEIGHT\"):\n    nce_weights = tf.Variable(\n                        tf.truncated_normal([vocabulary_size, embedding_size],\n                        stddev=1.0 / math.sqrt(embedding_size)))\n    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/cpu:0'):\n    # Loss function \n    num_sampled = 64        # Number of negative examples to sample. \n    \n    loss = tf.reduce_mean(\n                 tf.nn.nce_loss(weights=nce_weights,\n                 biases=nce_biases,\n                 labels=train_labels,\n                 inputs=embed,\n                 num_sampled=num_sampled,\n                 num_classes=vocabulary_size))\n\n    # Optimizer\n    optm = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n    \n    # Similarity measure (important)\n    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n    normalized_embeddings = embeddings / norm\n    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n    siml = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(normalized_embeddings.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train! \nsess = tf.Session()\nsess.run(tf.initialize_all_variables())\n#summary_writer = tf.summary.FileWriter('./w2v_train', graph=sess.graph)\naverage_loss = 0\n\nnum_steps = 10001\nfor iter in xrange(num_steps):\n    batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n    feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n    _, loss_val = sess.run([optm, loss], feed_dict=feed_dict)\n    average_loss += loss_val\n    \n    if iter % 2000 == 0:\n        average_loss /= 2000\n        print (\"Average loss at step %d is %.3f\" % (iter, average_loss)) \n    \n    if iter % 10000 == 0:\n        siml_val = sess.run(siml)\n        for i in xrange(valid_size): # Among valid set \n            valid_word = reverse_dictionary[valid_examples[i]]\n            top_k = 6 # number of nearest neighbors\n            nearest = (-siml_val[i, :]).argsort()[1:top_k+1]\n            log_str = \"Nearest to '%s':\" % valid_word\n            for k in xrange(top_k):\n                close_word = reverse_dictionary[nearest[k]] \n                log_str = \"%s '%s',\" % (log_str, close_word)\n            print(log_str) \n            \n# Final embeding \nfinal_embeddings = sess.run(normalized_embeddings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_points = 100\ntsne = TSNE(perplexity=10, n_components=2, init='pca', n_iter=5000)\ntwo_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])\n\ndef plot(embeddings, labels):\n    assert embeddings.shape[0] >= len(labels), 'More labels than embeddings'\n    plt.figure(figsize=(15,15))  # in inches\n    for i, label in enumerate(labels):\n        x, y = embeddings[i,:]\n        plt.scatter(x, y, color=['blue'])\n        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n                       ha='right', va='bottom')\n    plt.show()\n\nwords = [reverse_dictionary[i] for i in range(1, num_points+1)]\nplot(two_d_embeddings, words)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save only Numpy matrices to 'word2vec.npz'\nnp.savez(filename[0:-4] +'_word2vec_' + str(embedding_size), word_count=word_count, dictionary=dictionary, reverse_dictionary=reverse_dictionary, word_embeddings=final_embeddings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using Cosine Similiarty to find k-nearest neighbors of target vector $a$\n\n$\\frac{a \\cdot b}{|a||b|} = cos \\theta$, where $|a|$ can be omitted.","metadata":{}},{"cell_type":"code","source":"# Test numpy word2vectors\nK = 10\ntarget = 'drunk'\nscores = final_embeddings[dictionary[target]].dot(final_embeddings.transpose())\nscores = scores / np.linalg.norm(final_embeddings, axis=1)\nk_neighbors = (-scores).argsort()[0:K+1]  \n\nprint('The nearest neighbors of', target, 'are:')\nfor k in k_neighbors:\n    print(reverse_dictionary[k], ' ', scores[k])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (Optional) You can download the embedding vectors and upload to projector.tensorflow.org\nout_v = open('vecs.tsv', 'w', encoding='utf-8')\nout_m = open('meta.tsv', 'w', encoding='utf-8')\nfor num, word in enumerate(dictionary):\n  vec = final_embeddings[num] # skip 0, it's padding.\n  out_m.write(word + \"\\n\")\n  out_v.write('\\t'.join([str(x) for x in vec]) + \"\\n\")\nout_v.close()\nout_m.close()\n\nfrom IPython.display import FileLink\nFileLink('vecs.tsv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('meta.tsv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Upload vecs.tsv and meta.tsv to projector.tensorflow.org \n\n![](http://www.aiotlab.org/teaching/dl_app/drunk_neighbors.png)","metadata":{}},{"cell_type":"code","source":"# Download your word embedding vectors\nfrom IPython.display import FileLink\nFileLink(filename[0:-4] +'_word2vec_' + str(embedding_size) + '.npz')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}