{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is a simple exploration of the train label \n\n* Train a simple Word2Vec model for 2 dimensions and get the average embeddings of each the labels\n* Cluster them by DBSCAN\n* Visualize each cluster using scatter plot","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom gensim.models import word2vec\nfrom sklearn.cluster import KMeans, DBSCAN\n\nimport plotly.offline as pyo\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:27:21.598320Z","iopub.execute_input":"2021-06-18T08:27:21.598842Z","iopub.status.idle":"2021-06-18T08:27:25.423516Z","shell.execute_reply.started":"2021-06-18T08:27:21.598722Z","shell.execute_reply":"2021-06-18T08:27:25.422277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:27:33.130159Z","iopub.execute_input":"2021-06-18T08:27:33.130589Z","iopub.status.idle":"2021-06-18T08:27:33.332070Z","shell.execute_reply.started":"2021-06-18T08:27:33.130554Z","shell.execute_reply":"2021-06-18T08:27:33.330578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Form corpus","metadata":{}},{"cell_type":"code","source":"# Frame the corpus which is the unique train labels\n\ntrain_unique_label = np.unique(train_df.cleaned_label.values.astype('str'))\ncorpus = [label.split() for label in train_unique_label]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:27:37.441541Z","iopub.execute_input":"2021-06-18T08:27:37.441951Z","iopub.status.idle":"2021-06-18T08:27:37.495063Z","shell.execute_reply.started":"2021-06-18T08:27:37.441916Z","shell.execute_reply":"2021-06-18T08:27:37.493805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word2Vec Train function","metadata":{}},{"cell_type":"code","source":"def train_w2vec(corpus, dim, epochs):\n    '''\n    Function to train a simple word2Vec model\n    '''\n    model = word2vec.Word2Vec(vector_size = dim, min_count= 1)\n    model.build_vocab(corpus)\n    model.train(corpus, epochs=epochs, total_examples=model.corpus_count)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:27:40.321943Z","iopub.execute_input":"2021-06-18T08:27:40.322401Z","iopub.status.idle":"2021-06-18T08:27:40.329227Z","shell.execute_reply.started":"2021-06-18T08:27:40.322357Z","shell.execute_reply":"2021-06-18T08:27:40.327664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mxnet import nd\nfrom mxnet.contrib import text\n\ndef download_model():\n    print(text.embedding.get_pretrained_file_names('glove'))\n    glove_6b50d = text.embedding.create(\n        'glove', pretrained_file_name='glove.6B.50d.txt')\n    return glove_6b50d\n\nmodel = download_model()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:27:51.727141Z","iopub.execute_input":"2021-06-18T08:27:51.727521Z","iopub.status.idle":"2021-06-18T08:29:21.166582Z","shell.execute_reply.started":"2021-06-18T08:27:51.727488Z","shell.execute_reply":"2021-06-18T08:29:21.164147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function to Get average embeddings","metadata":{}},{"cell_type":"code","source":"def get_avg_label_embedding(corpus, dim, model):\n    '''\n    Function to get the average word embeddings when the label is more than 1 word\n    '''\n\n    label_embedding = []\n    for label in corpus:\n        embedding_sum = np.zeros(shape=(dim, ))\n        for tok in label:\n            temp = model.get_vecs_by_tokens([tok]).asnumpy().reshape(-1)\n\n            embedding_sum += temp\n            \n        avg_embedding = embedding_sum/len(label)\n        label_embedding.append(avg_embedding.tolist())\n    \n    label_embedding = np.array(label_embedding)\n    \n    return label_embedding","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:31:53.594749Z","iopub.execute_input":"2021-06-18T08:31:53.595174Z","iopub.status.idle":"2021-06-18T08:31:53.603387Z","shell.execute_reply.started":"2021-06-18T08:31:53.595140Z","shell.execute_reply":"2021-06-18T08:31:53.602023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and get embedding","metadata":{}},{"cell_type":"code","source":"# Train the word2Vec model for few epochs\ndim = 50\nlabel_embedding = get_avg_label_embedding(corpus, dim, model)\n\nlabel_embedding.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:31:56.304565Z","iopub.execute_input":"2021-06-18T08:31:56.304999Z","iopub.status.idle":"2021-06-18T08:31:56.645486Z","shell.execute_reply.started":"2021-06-18T08:31:56.304960Z","shell.execute_reply":"2021-06-18T08:31:56.644538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def euclidean_distance(A, B, axis=None):\n    \"\"\" A, B are array or matrix\n    \"\"\"\n    return np.sqrt( np.power(A - B, 2).sum(axis=axis) )\n\ndef rand_k_centroids(data, k=3):\n    m, n = np.shape(data)\n    centroids = np.mat( np.zeros((k, n)) )\n\n    min_j = np.min(data, axis=0)\n    rang_j = np.max(data, axis=0) - min_j\n\n    centroids = np.tile(min_j, (k, 1)) + \\\n        np.multiply(np.tile(rang_j, (k, 1)),\n                    np.random.rand(k, n))\n\n#    for j in range(n):\n#        min_j   = np.min(data[:, j])\n#        range_j = np.max(data[:, j]) - min_j\n#        centroids[:, j] = min_j + range_j * np.random.rand(k, 1)\n    return centroids\n\ndef kmeans(data, k, calculate_distance=euclidean_distance, create_centroids=rand_k_centroids):\n    \"\"\" k means only converge to local minimum,\n        the result will easily affect by initial centroids\n        Theoretically, the result of clustering will shake, but happens rarely.\n        two problems:\n            1. one cluster can split. which one?\n                a. the cluster with biggest SSE(sum of squared error) until cluster growth to k;\n                b. after the cluster splitting, sum of all clusters' SSE is minimum\n            2. two cluster can merge. which two?\n                a. two nearest centroids;\n                b. after the two centroids merging, sum of all clusters' SSE is minimum\n        After analysing the two problems, we got bisecting k means.\n    \"\"\"\n    m, n = np.shape(data)\n    # initial with -1, in case the compare changed conflict\n    # 簇分配矩阵 [assign data points to a centroid, holds Sum of Squared Error to each point]\n    cluster_assignment = np.mat( -np.ones((m, 2)) )\n    centroids = create_centroids(data, k)\n    changed = True\n\n    while changed:\n        changed = False\n\n        for i in range(m): # for each data point, assign it to the closest centroid\n            min_distance = -1; min_index = -2\n\n            for j in range(k):\n                distance = calculate_distance( data[i, :], centroids[j, :] )\n                if distance < min_distance or min_distance == -1:\n                    min_distance = distance; min_index = j\n\n            if cluster_assignment[i, 0] != min_index: changed = True\n            cluster_assignment[i, :] = (min_index, min_distance**2)\n\n        for centre in range(k): # recalculate centroids\n            points_in_cluster = data[ np.nonzero(cluster_assignment[:, 0].A == centre)[0] ] # get all points in this cluster\n            # assign centroid to mean of all points in this cluster\n            # if points_in_cluster is empty, this centroid is np.nan, warnings is printed\n            centroids[centre, :] = np.mean(points_in_cluster, axis=0)\n    return centroids, cluster_assignment\n\ndef bisecting_kmeans(data, k, calculate_distance=euclidean_distance):\n    \"\"\" We start with one cluster, split it to k.\n        Split the cluster which can decrease the SSE(sum of squared error) most.\n        converge at global minimum\n    \"\"\"\n    m, n = np.shape(data)\n    centroid = np.mean(data, axis=0).tolist()[0]\n    centroids = [centroid] # list with one centroid\n\n    cluster_assignment = np.mat( np.zeros((m, 2)) )\n    for i in range(m): # calculate initial SSE\n        cluster_assignment[i, 1] = calculate_distance(centroid, data[i, :]) ** 2\n\n    while(len(centroids) < k):\n        lowest_SSE = -1\n        for i, _ in enumerate(centroids):\n            subcluster = data[np.nonzero(cluster_assignment[:, 0].A == i)[0], :] # get all points in cluster i\n            # this cluster has no point, if deleted this centroid, centroids length may never reach k\n            # if len(subcluster) == 0: del(centroids[i]); continue\n\n            subcentroids, subcluster_assignment = kmeans(subcluster, 2, calculate_distance)\n            if np.any( np.isnan(subcentroids) ) == True:\n                # Do 2means again if one cluster has no point, or else will generate zero point centroid\n                # that means this subcluster don't need to be split\n                subcentroids, subcluster_assignment = kmeans(subcluster, 2, calculate_distance)\n                if np.isnan( np.sum(subcentroids) ) == True:\n                    continue\n\n            subSSE = np.sum( subcluster_assignment[:, 1] )\n            non_subSSE = np.sum( cluster_assignment[np.nonzero(cluster_assignment[:, 0].A != i)[0], 1] )\n            if (subSSE + non_subSSE) < lowest_SSE or lowest_SSE == -1:\n                lowest_SSE = subSSE + non_subSSE\n                best_split_centre = i\n                best_subcentroids = subcentroids\n                best_subcluster_assignment = subcluster_assignment\n\n        if lowest_SSE == -1: break # no suitable split in centroids\n        print(\"{} centroids SSE: {}\".format(len(centroids)+1, lowest_SSE))\n        # len(centroids) larger than any index in centroids, need use to assign first\n        # if best_split_centre is 0, or len(centroids) is 1, they can affect best_subcluster_assignment without intermediate value\n        row1 = np.nonzero(best_subcluster_assignment[:, 0].A == 1)[0]\n        row0 = np.nonzero(best_subcluster_assignment[:, 0].A == 0)[0]\n        best_subcluster_assignment[row1, 0] = len(centroids)\n        best_subcluster_assignment[row0, 0] = best_split_centre\n        cluster_assignment[np.nonzero(cluster_assignment[:, 0].A == best_split_centre)[0], :] = best_subcluster_assignment\n\n        centroids[best_split_centre] = best_subcentroids[0].tolist()[0] # replace a centroid with two better centroids\n        centroids.append(best_subcentroids[1].tolist()[0])\n    return np.mat(centroids), cluster_assignment","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:31:59.003141Z","iopub.execute_input":"2021-06-18T08:31:59.003610Z","iopub.status.idle":"2021-06-18T08:31:59.031418Z","shell.execute_reply.started":"2021-06-18T08:31:59.003571Z","shell.execute_reply":"2021-06-18T08:31:59.029889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cluster the data using kmeans","metadata":{}},{"cell_type":"code","source":"centroids, ca = bisecting_kmeans(label_embedding, 100)\n\nprint(f\"centroids : {centroids}[{centroids.shape}], ca: {ca}[{ca.shape}]\")","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:32:04.264768Z","iopub.execute_input":"2021-06-18T08:32:04.265185Z","iopub.status.idle":"2021-06-18T08:32:08.365401Z","shell.execute_reply.started":"2021-06-18T08:32:04.265150Z","shell.execute_reply":"2021-06-18T08:32:08.364127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize\n-> However over the points for name of the dataset","metadata":{}},{"cell_type":"code","source":"# Set notebook mode to work in offline\npyo.init_notebook_mode()\n\n# Create traces\ntrace0 = px.scatter(x = label_embedding[:, 0], \n                    y = label_embedding[:, 1], \n                    color = ca[:, 0].astype(np.int32).reshape(-1).tolist()[0], \n                    hover_name=train_unique_label\n)\n\ntrace0.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:32:14.811109Z","iopub.execute_input":"2021-06-18T08:32:14.811572Z","iopub.status.idle":"2021-06-18T08:32:16.495971Z","shell.execute_reply.started":"2021-06-18T08:32:14.811523Z","shell.execute_reply":"2021-06-18T08:32:16.495146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With a quick hover on the points it looks like the cluster are good and similar domains are grouped together with few data points which are not clustered right. However you can play around with it by trying\n\n* Different epochs for Word2Vec, (tried)\n* Train for more Dimensions and reduce them using t-sne or PCA\n* Play with different cluster distance mertrics for DBSCAN clusters or different cluster algorithms (tried)","metadata":{}},{"cell_type":"markdown","source":"The label embedding is the average of all the word embedding in this label. Don't know whether this method can represent the label embedding?\n\nIf the answer is yes, I think the conlusion is the labels can not be clustered.","metadata":{}}]}