{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\nimport collections\nimport math\nimport os\nimport numpy as np\nimport random\nfrom six.moves import xrange\nfrom sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nif tf.__version__[0] == '2':\n    # Using TensorFlow 1.x to train our word2vec\n    import tensorflow.compat.v1 as tf\n    tf.disable_v2_behavior()\n%matplotlib inline  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_folder  = \"../input/109-1-ntut-dl-app-hw2\"\nfilename   = \"tag_list.txt\"  # Hashtag list\nvocabulary_size = 990","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size     = 128\nembedding_size = 64       # Dimension of the embedding vector.\nskip_window    = 1         # How many words to consider left and right.\nnum_skips      = 2         # How many times to reuse an input ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random validation set to sample nearest neighbors.\nvalid_size     = 32        # Random set of words to evaluate similarity \nvalid_window   = 200       # Only pick validation samples in the top 200\nvalid_examples = np.random.choice(valid_window, valid_size, replace=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_path   = os.path.join(data_folder, filename)\nwith open(file_path, 'r', encoding=\"utf-8\") as f:\n    words = f.read().split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_count = [['UNK', -1]] \nword_count.extend(collections.Counter(words)\n             .most_common(vocabulary_size - 1)) # -1 is for UNK \nprint (\"%s\" % (word_count[0:10]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create word -> wordID dictionary\ndictionary = dict() \nfor word, _ in word_count:\n    dictionary[word] = len(dictionary)\n\n# Create reverse dictionary (wordID -> word)\nreverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert word into wordID, and count unused words (UNK)\ndata = list()\nunk_count = 0\nfor word in words:\n    if word in dictionary:\n        index = dictionary[word]\n    else:\n        index = 0  # dictionary['UNK']\n        unk_count += 1\n    data.append(index)\nword_count[0][1] = unk_count\n# del words  # Hint to reduce memory.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\"Most common words (+UNK) are: %s\" % (word_count[:10]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\"Sample data corresponds to\\n__________________\")\nfor i in range(10):\n    print (\"%d->%s\" % (data[i], reverse_dictionary[data[i]]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data batch generator\ndata_index = 0\ndef generate_batch(batch_size, num_skips, skip_window):\n    global data_index\n    assert batch_size % num_skips == 0\n    assert num_skips <= 2 * skip_window\n    batch  = np.ndarray(shape=(batch_size),    dtype=np.int32)\n    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n    span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n    buffer = collections.deque(maxlen=span)\n    for _ in range(span):\n        buffer.append(data[data_index])\n        data_index = (data_index + 1) % len(data)\n    for i in range(batch_size // num_skips): # '//' makes the result an integer, e.g., 7//3 = 2\n        target = skip_window\n        targets_to_avoid = [ skip_window ]\n        for j in range(num_skips):\n            while target in targets_to_avoid:\n                target = random.randint(0, span - 1)\n            targets_to_avoid.append(target)\n            batch[i * num_skips + j] = buffer[skip_window]\n            labels[i * num_skips + j, 0] = buffer[target]\n        buffer.append(data[data_index])\n        data_index = (data_index + 1) % len(data)\n    return batch, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Construct the word2vec model \ntrain_inputs   = tf.placeholder(tf.int32, shape=[batch_size])   \ntrain_labels   = tf.placeholder(tf.int32, shape=[batch_size, 1])\nvalid_dataset  = tf.constant(valid_examples, dtype=tf.int32)\n\n# Look up embeddings for inputs. (vocabulary_size = 50,000)\nwith tf.variable_scope(\"EMBEDDING\"):\n    with tf.device('/cpu:0'):\n        embeddings = tf.Variable(\n            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n        embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n    \n# Construct the variables for the NCE loss\nwith tf.variable_scope(\"NCE_WEIGHT\"):\n    nce_weights = tf.Variable(\n                        tf.truncated_normal([vocabulary_size, embedding_size],\n                        stddev=1.0 / math.sqrt(embedding_size)))\n    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tf.device('/cpu:0'):\n    # Loss function \n    num_sampled = 64        # Number of negative examples to sample. \n    \n    loss = tf.reduce_mean(\n                 tf.nn.nce_loss(weights=nce_weights,\n                 biases=nce_biases,\n                 labels=train_labels,\n                 inputs=embed,\n                 num_sampled=num_sampled,\n                 num_classes=vocabulary_size))\n\n    # Optimizer\n    optm = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n    \n    # Similarity measure (important)\n    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n    normalized_embeddings = embeddings / norm\n    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n    siml = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(normalized_embeddings.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train! \nsess = tf.Session()\nsess.run(tf.initialize_all_variables())\n#summary_writer = tf.summary.FileWriter('./w2v_train', graph=sess.graph)\naverage_loss = 0\n\nnum_steps = 10001\nfor iter in xrange(num_steps):\n    batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n    feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n    _, loss_val = sess.run([optm, loss], feed_dict=feed_dict)\n    average_loss += loss_val\n    \n    if iter % 2000 == 0:\n        average_loss /= 2000\n        print (\"Average loss at step %d is %.3f\" % (iter, average_loss)) \n    \n    if iter % 10000 == 0:\n        siml_val = sess.run(siml)\n        for i in xrange(valid_size): # Among valid set \n            valid_word = reverse_dictionary[valid_examples[i]]\n            top_k = 6 # number of nearest neighbors\n            nearest = (-siml_val[i, :]).argsort()[1:top_k+1]\n            log_str = \"Nearest to '%s':\" % valid_word\n            for k in xrange(top_k):\n                close_word = reverse_dictionary[nearest[k]] \n                log_str = \"%s '%s',\" % (log_str, close_word)\n            print(log_str) \n            \n# Final embeding \nfinal_embeddings = sess.run(normalized_embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_points = 100\ntsne = TSNE(perplexity=10, n_components=2, init='pca', n_iter=5000)\ntwo_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])\n\ndef plot(embeddings, labels):\n    assert embeddings.shape[0] >= len(labels), 'More labels than embeddings'\n    plt.figure(figsize=(15,15))  # in inches\n    for i, label in enumerate(labels):\n        x, y = embeddings[i,:]\n        plt.scatter(x, y, color=['blue'])\n        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n                       ha='right', va='bottom')\n    plt.show()\n\nwords = [reverse_dictionary[i] for i in range(1, num_points+1)]\nplot(two_d_embeddings, words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save only Numpy matrices to 'word2vec.npz'\nnp.savez(filename[0:-4] +'_word2vec_' + str(embedding_size), word_count=word_count, dictionary=dictionary, reverse_dictionary=reverse_dictionary, word_embeddings=final_embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test numpy word2vectors\nK = 10\ntarget = 'drunk'\nscores = final_embeddings[dictionary[target]].dot(final_embeddings.transpose())\nscores = scores / np.linalg.norm(final_embeddings, axis=1)\nk_neighbors = (-scores).argsort()[0:K+1]  \n\nprint('The nearest neighbors of', target, 'are:')\nfor k in k_neighbors:\n    print(reverse_dictionary[k], ' ', scores[k])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# (Optional) You can download the embedding vectors and upload to projector.tensorflow.org\nout_v = open('vecs.tsv', 'w', encoding='utf-8')\nout_m = open('meta.tsv', 'w', encoding='utf-8')\nfor num, word in enumerate(dictionary):\n  vec = final_embeddings[num] # skip 0, it's padding.\n  out_m.write(word + \"\\n\")\n  out_v.write('\\t'.join([str(x) for x in vec]) + \"\\n\")\nout_v.close()\nout_m.close()\n\nfrom IPython.display import FileLink\nFileLink('vecs.tsv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('meta.tsv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Download your word embedding vectors\nfrom IPython.display import FileLink\nFileLink(filename[0:-4] +'_word2vec_' + str(embedding_size) + '.npz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(data_folder) # For importing utiliy.py\n\nimport keras\nimport numpy as np\nfrom utility import *\n\nfrom keras import models, layers, Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_FEAT_FILE = data_folder + '/harrison_features.npz'\n#embedding_size = 64\nTAG2VEC_FILE = 'tag_list_word2vec_' + str(embedding_size) + '.npz'\nMODEL_NAME = 'im_hashtag_model.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Hashtag-to-Vector file\nfrom packaging import version\nt2v_data = None\nif version.parse(np.__version__) > version.parse(\"1.16.2\"):\n    t2v_data = np.load(TAG2VEC_FILE, allow_pickle=True) # word_count, dictionary, reverse_dictionary, word_embeddings\nelse:\n    t2v_data = np.load(TAG2VEC_FILE) # word_count, dictionary, reverse_dictionary, word_embeddings\n    \ntag_dictionary = t2v_data['dictionary'].tolist()\ntag_reverse_dict = t2v_data['reverse_dictionary'].tolist()\ntag_embeddings = t2v_data['word_embeddings']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load ImageNet features\nfeat_data = np.load(IMAGE_FEAT_FILE)\nimnet_fc_feats = feat_data['imagenet_fc_layers']\nplace_fc_feats = feat_data['places365_fc_layers']\nimage_list = feat_data['image_list']\ntag_strings = feat_data['hashtag_list']\ntrain_indices = feat_data['train_indices']\ntest_indices = feat_data['test_indices']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get HARRISON labels\nlabels = img_paths_to_labels(image_list)\n\n# Get word vector of each image\nimage_t2v = []\nfor label in labels:\n    if label in tag_dictionary:\n        tagID = tag_dictionary[label]\n        image_t2v.append(tag_embeddings[tagID])\n    else:\n        print(label, \"doesn't exist!\")\n\nimage_t2v_mat = np.array(image_t2v)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Split data into train/test set\ntrain_x = imnet_fc_feats[train_indices, :]\ntest_x = imnet_fc_feats[test_indices, :]\ntrain_y = image_t2v_mat[train_indices, :]\ntest_y = image_t2v_mat[test_indices, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Your Neural Network model\n#-------------\nmodel = models.Sequential()\nmodel.add(layers.Dense(128, activation='linear', input_shape=(2048,)))\nmodel.add(layers.Dense(embedding_size))\n#-------------\nmodel.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])\n\nhistory = model.fit(train_x, train_y, epochs=20, batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(MODEL_NAME)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#### Evaluate DeViSE Model ####\nmodel.evaluate(test_x, test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = model.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert HARRISON tag ground-truth string into wordID\ntag_list, miss_tags = tag_string_to_w2v_id(tag_strings, tag_dictionary)\ntest_tag_list = []\nfor tid in test_indices:\n    test_tag_list.append(tag_list[tid])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate K-Nearest Neighbor of predicted y (tag-vector)\nKNN = 10\nknn_list = []\nfor pred in pred_y:\n    knn_list.append(get_knn(pred, tag_embeddings, KNN))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"recall_rate, recall_list = cal_recall_rate(knn_list, test_tag_list)\nprint('The recall rate of our model is', recall_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_labels = []\nfor tid in test_indices:\n    #test_tag_list.append(tag_list[tid])\n    test_labels.append(labels[tid])\n    \nrecall_rates, _ = cal_class_recall(test_labels, recall_list, test_tag_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(15,40))\nsns.set(style=\"whitegrid\")\nax = sns.barplot(x=list(recall_rates.values()), y=list(recall_rates.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss, result_list = cal_hamming_loss(knn_list, test_tag_list)\nprint('Hamming loss (lower is better): %s' % loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"save_kaggle_hamming_loss_csv('predict_4_kaggle.csv', result_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\nlocfile = \"./predict_4_kaggle.csv\"\nloczip = \"./predict_4_kaggle.zip\"\nzip = zipfile.ZipFile(loczip, \"w\", zipfile.ZIP_DEFLATED)\nzip.write(locfile)\nzip.close()\n# I zip it because that Kaggle can only download file less than 2 MB from HTML. ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\ndef create_download_link(title = \"Ashish zip file\", filename = \"predict_4_kaggle.zip\"):  \n    html = '<a href={filename}>{title}</a>'\n    html = html.format(title=title,filename=filename)\n    return HTML(html)\n\n# create a link to download the dataframe which was saved with .to_csv method\ncreate_download_link(filename='predict_4_kaggle.zip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}