{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom gensim.models import word2vec\nfrom sklearn.cluster import KMeans, DBSCAN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T11:26:23.172742Z","iopub.execute_input":"2021-06-21T11:26:23.173415Z","iopub.status.idle":"2021-06-21T11:26:24.966388Z","shell.execute_reply.started":"2021-06-21T11:26:23.17327Z","shell.execute_reply":"2021-06-21T11:26:24.965263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\n# test_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:26:24.968126Z","iopub.execute_input":"2021-06-21T11:26:24.96844Z","iopub.status.idle":"2021-06-21T11:26:25.127584Z","shell.execute_reply.started":"2021-06-21T11:26:24.968412Z","shell.execute_reply":"2021-06-21T11:26:25.126512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:26:31.107991Z","iopub.execute_input":"2021-06-21T11:26:31.108357Z","iopub.status.idle":"2021-06-21T11:26:31.117766Z","shell.execute_reply.started":"2021-06-21T11:26:31.10832Z","shell.execute_reply":"2021-06-21T11:26:31.116137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:26:39.274686Z","iopub.execute_input":"2021-06-21T11:26:39.275039Z","iopub.status.idle":"2021-06-21T11:26:39.281413Z","shell.execute_reply.started":"2021-06-21T11:26:39.275008Z","shell.execute_reply":"2021-06-21T11:26:39.280364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.index","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:26:49.392249Z","iopub.execute_input":"2021-06-21T11:26:49.392642Z","iopub.status.idle":"2021-06-21T11:26:49.399114Z","shell.execute_reply.started":"2021-06-21T11:26:49.392609Z","shell.execute_reply":"2021-06-21T11:26:49.39802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:27:11.627709Z","iopub.execute_input":"2021-06-21T11:27:11.62808Z","iopub.status.idle":"2021-06-21T11:27:11.650596Z","shell.execute_reply.started":"2021-06-21T11:27:11.628049Z","shell.execute_reply":"2021-06-21T11:27:11.649674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:27:16.930247Z","iopub.execute_input":"2021-06-21T11:27:16.930759Z","iopub.status.idle":"2021-06-21T11:27:16.948504Z","shell.execute_reply.started":"2021-06-21T11:27:16.930725Z","shell.execute_reply":"2021-06-21T11:27:16.947204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Importing NLTK librariries so that to clean the unstructured language data\nimport nltk\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\nfrom nltk import word_tokenize\nfrom nltk import pos_tag\nimport spacy\nenglish_spacy = spacy.load('en_core_web_lg')\nfrom spacy import displacy\nimport string\nimport re\nnltk.download('stopwords', quiet=True)\nstopwords = nltk.corpus.stopwords.words('english')\n\n#removing punctuations and stopwords from unstructured langugae data \ndef stopwordsPunctuation_detacher(syntax):\n    syntax = re.sub(r'[^\\w\\s]','',syntax)\n    syntax = [lexUnit.lower() for lexUnit in syntax.lower().split() if lexUnit not in stopwords]\n    syntax=' '.join(syntax)\n    syntax.translate(str.maketrans('','', string.punctuation))\n    english_sytax = english_spacy(syntax)\n    new_syntax = ' '\n    absence=0\n    for lex_unit in english_sytax:\n        if (lex_unit.is_stop == absence):\n            new_syntax = new_syntax + ' ' + str(lex_unit)\n    return new_syntax\n\n#removing digits, webLinks from unstructured langugae data and making all words lower case.\ndef textCleaning_engine(syntax):\n    syntax = syntax.lower()\n    syntax = re.sub('\\[.*?\\]', '', syntax)\n    syntax = re.sub('https?://\\S+|www\\.\\S+', '', syntax)\n    syntax = re.sub('<.*?>+', '', syntax)\n    syntax = re.sub('[%s]' % re.escape(string.punctuation), '', syntax)\n    syntax = re.sub('\\n', '', syntax)\n    syntax = re.sub('\\w*\\d\\w*', '', syntax)\n    clean_syntax = re.sub(r'\\d+','',syntax)\n    return clean_syntax\n#Combining the two functions above in one function\ndef ready_sytaxt(syntax):\n    syntax = syntax.lower()\n    syntax = stopwordsPunctuation_detacher(syntax)\n    syntax = textCleaning_engine(syntax)\n    return syntax\n\ntrain_df['dataset_title'] = train_df['dataset_title'].apply(ready_sytaxt)\ntrain_df['pub_title'] = train_df['pub_title'].apply(ready_sytaxt)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:27:30.632082Z","iopub.execute_input":"2021-06-21T11:27:30.632477Z","iopub.status.idle":"2021-06-21T11:33:24.222764Z","shell.execute_reply.started":"2021-06-21T11:27:30.632445Z","shell.execute_reply":"2021-06-21T11:33:24.22158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_title_LIST=train_df['dataset_title'].to_list()\n\nNational_Education_Longitudinal_Study=[]\nBeginning_Postsecondary_Student=[]\nAgricultural_Resource_Management_Survey=[]\nBaltimore_Longitudinal_StudyOfAging=[]\nCoastal_Change_Analysis_Program=[]\nSea_Lake_Overland_Surges_fromHurricanes=[]\nNOAA_Tide_Gauge=[]\nfor i in dataset_title_LIST:\n    if i=='National Education Longitudinal Study':\n        National_Education_Longitudinal_Study.append(i)\n    elif i=='NOAA Tide Gauge':\n        NOAA_Tide_Gauge.append(i)\n    elif i=='Sea, Lake, and Overland Surges from Hurricanes':\n        Sea_Lake_Overland_Surges_fromHurricanes.append(i)\n    elif i=='Coastal Change Analysis Program':\n        Coastal_Change_Analysis_Program.append(i)\n    elif i==\"Baltimore Longitudinal Study of Aging (BLSA) \":\n        Baltimore_Longitudinal_StudyOfAging.append(i)\n    elif i==\"Agricultural Resource Management Survey \":\n        Agricultural_Resource_Management_Survey.append(i)\n    else:\n        Beginning_Postsecondary_Student.append(i)\n        \n# print(National_Education_Longitudinal_Study)\n# Beginning_Postsecondary_Student \n# Agricultural_Resource_Management_Survey \n# Baltimore_Longitudinal_StudyOfAging \n# Coastal_Change_Analysis_Program \n# Sea_Lake_Overland_Surges_fromHurricanes\n# NOAA_Tide_Gauge\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:33:24.224912Z","iopub.execute_input":"2021-06-21T11:33:24.225225Z","iopub.status.idle":"2021-06-21T11:33:24.240514Z","shell.execute_reply.started":"2021-06-21T11:33:24.225193Z","shell.execute_reply":"2021-06-21T11:33:24.239281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ready_label=train_df['cleaned_label'].values.astype('str')\ntrain_uniqueLabels = np.unique(ready_label)\nlabels = [lbl.split() for lbl in train_uniqueLabels]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:33:24.242098Z","iopub.execute_input":"2021-06-21T11:33:24.242517Z","iopub.status.idle":"2021-06-21T11:33:24.301678Z","shell.execute_reply.started":"2021-06-21T11:33:24.242483Z","shell.execute_reply":"2021-06-21T11:33:24.300387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainANDbuildVocab(label, dimension, epochs):\n    word2vec_model = word2vec.Word2Vec(vector_size = dimension, min_count= 1)\n    word2vec_model.build_vocab(labels)\n    word2vec_model.train(labels, epochs=epochs, total_examples=word2vec_model.corpus_count)\n    return word2vec_model\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:37:12.731482Z","iopub.execute_input":"2021-06-21T11:37:12.731851Z","iopub.status.idle":"2021-06-21T11:37:12.7375Z","shell.execute_reply.started":"2021-06-21T11:37:12.73182Z","shell.execute_reply":"2021-06-21T11:37:12.736284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# glovePretrainedModel = pd.read_csv('../input/glove6b50dtxt/glove.6B.50d.txt', sep=\" \", quoting=3, header=None, index_col=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:37:13.783603Z","iopub.execute_input":"2021-06-21T11:37:13.784161Z","iopub.status.idle":"2021-06-21T11:37:13.788637Z","shell.execute_reply.started":"2021-06-21T11:37:13.784111Z","shell.execute_reply":"2021-06-21T11:37:13.787541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mxnet import nd\nfrom mxnet.contrib import text\nimport pickle\n\ndef download_gloveAlgorithm():\n    print(text.embedding.get_pretrained_file_names('glove'))\n    obtain_vector_representations = text.embedding.create('glove', \n                                                          pretrained_file_name='glove.6B.50d.txt')\n    return obtain_vector_representations\n    \n#     with open('glove.6B.50d.pkl', 'wb') as fp:\n#         pickle.dump(glovePretrainedModel, fp)\n\nglovePretrainedModel = download_gloveAlgorithm()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:04.584277Z","iopub.execute_input":"2021-06-21T11:40:04.584706Z","iopub.status.idle":"2021-06-21T11:40:17.133665Z","shell.execute_reply.started":"2021-06-21T11:40:04.58467Z","shell.execute_reply":"2021-06-21T11:40:17.132425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getAverage_attachedLabel(labels, dimension, attaching_index):\n    labelAttached_container = []\n    for token in labels:\n        sumOf_attachedLAbels = np.zeros(shape=(dimension, ))\n        for tkn in token:\n            getVecByToken = attaching_index.get_vecs_by_tokens([tkn]).asnumpy().reshape(-1)\n\n            sumOf_attachedLAbels += getVecByToken\n            \n        avg_embedding = sumOf_attachedLAbels/len(token)\n        labelAttached_container.append(avg_embedding.tolist())\n\n    labelAttached_container = np.array(labelAttached_container)\n    \n    return labelAttached_container","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:21.292591Z","iopub.execute_input":"2021-06-21T11:40:21.292937Z","iopub.status.idle":"2021-06-21T11:40:21.308354Z","shell.execute_reply.started":"2021-06-21T11:40:21.292906Z","shell.execute_reply":"2021-06-21T11:40:21.307433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averageAttachedLabel = getAverage_attachedLabel(labels, \n                                                50, \n                                                glovePretrainedModel)\naverageAttachedLabel.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:22.744278Z","iopub.execute_input":"2021-06-21T11:40:22.744865Z","iopub.status.idle":"2021-06-21T11:40:23.192526Z","shell.execute_reply.started":"2021-06-21T11:40:22.744808Z","shell.execute_reply":"2021-06-21T11:40:23.19124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def distance_betweenPoints(a_point, b_point, axis=None):#eucledean distance\n    return np.sqrt( np.power(a_point - b_point, 2).sum(axis=axis) )\n\ndef multiDimensionalAverageArray_ofTheCluster(element, cluster=3):#cetroid\n    mean, observation = np.shape(element)\n    centroids = np.mat( np.zeros((cluster, observation)) )\n\n    minimizeDimension = np.min(element, axis=0)\n    maximizeDimension = np.max(element, axis=0) - minimizeDimension\n\n    multiDimensionalAverage_ofTheCluster = np.tile(minimizeDimension, (cluster, 1)) + np.multiply(np.tile(maximizeDimension, (cluster, 1)), np.random.rand(cluster, observation))\n\n    return multiDimensionalAverage_ofTheCluster","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:23.732211Z","iopub.execute_input":"2021-06-21T11:40:23.732583Z","iopub.status.idle":"2021-06-21T11:40:24.888626Z","shell.execute_reply.started":"2021-06-21T11:40:23.732551Z","shell.execute_reply":"2021-06-21T11:40:24.887812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def k_meansClustering(element, cluster, calculateEuclideanDistance=distance_betweenPoints, createClusterMean=multiDimensionalAverageArray_ofTheCluster):\n    mean, observations = np.shape(element)\n    newContainer_forCluster = np.mat( -np.ones((mean, 2)) )\n    ClusterMean_centroid = createClusterMean(element, cluster)\n    modified = True\n\n    while modified:\n        modified = False\n\n        for each_element in range(mean): \n            minDistance = -1 \n            minIndex = -2\n\n            for each_subElement in range(cluster):\n                EuclideanDistanceCalculation = calculateEuclideanDistance( element[each_element, :], ClusterMean_centroid[each_subElement, :] )\n                if EuclideanDistanceCalculation < minDistance or minDistance == -1:\n                    minDistance = EuclideanDistanceCalculation; minIndex = each_subElement\n\n            if newContainer_forCluster[each_element, 0] != minIndex: modified = True\n            newContainer_forCluster[each_element, :] = (minIndex, minDistance**2)\n\n        for centreCluster in range(cluster):  \n            pointsPatternOfCluster = element[ np.nonzero(newContainer_forCluster[:, 0].A == centreCluster)[0] ]  \n            ClusterMean_centroid[centreCluster, :] = np.mean(pointsPatternOfCluster, axis=0)\n    return ClusterMean_centroid, newContainer_forCluster","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:25.633404Z","iopub.execute_input":"2021-06-21T11:40:25.633805Z","iopub.status.idle":"2021-06-21T11:40:25.644864Z","shell.execute_reply.started":"2021-06-21T11:40:25.633766Z","shell.execute_reply":"2021-06-21T11:40:25.643409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef divisiveHierarchyClustering_kMeans(element, cluster, calculateEuclideanDistance=distance_betweenPoints):\n    mean, observations = np.shape(element)\n    multiDimensionalAverage_centroid = np.mean(element, axis=0).tolist()[0]\n    multiDimensionalAverage_cluster = [multiDimensionalAverage_centroid] \n\n    newContainer_forCluster = np.mat( np.zeros((mean, 2)) )\n    for each_element in range(mean):\n        newContainer_forCluster[each_element, 1] = calculateEuclideanDistance(multiDimensionalAverage_centroid, element[each_element, :]) ** 2\n\n    while(len(multiDimensionalAverage_cluster) < cluster):\n        lowest_SumOFSquaredError = -1\n        for each_element, _ in enumerate(multiDimensionalAverage_cluster):\n            subCluster = element[np.nonzero(newContainer_forCluster[:, 0].A == each_element)[0], :]  \n\n            subCentroids, subContainer_forCluster = k_meansClustering(subCluster, 2, calculateEuclideanDistance)\n            if np.any( np.isnan(subCentroids) ) == True:\n                subCentroids, subContainer_forCluster = k_meansClustering(subCluster, 2, calculateEuclideanDistance)\n                if np.isnan( np.sum(subCentroids) ) == True:\n                    continue\n\n            subSquaredError = np.sum( subContainer_forCluster[:, 1] )\n            nonZero_subSquaredError = np.sum( newContainer_forCluster[np.nonzero(newContainer_forCluster[:, 0].A != each_element)[0], 1] )\n            if (subSquaredError + nonZero_subSquaredError) < lowest_SumOFSquaredError or lowest_SumOFSquaredError == -1:\n                lowest_SumOFSquaredError = subSquaredError + nonZero_subSquaredError\n                optimalSplit_centre = each_element\n                optimal_subcentroids = subCentroids\n                optimalContainer_forCluster = subContainer_forCluster\n\n        if lowest_SumOFSquaredError == -1: break \n        print(\"{} multiDimensionalAverage_cluster of SumOFSquaredError: {}\".format(len(multiDimensionalAverage_cluster)+1, lowest_SumOFSquaredError))\n        indx1 = np.nonzero(optimalContainer_forCluster[:, 0].A == 1)[0]\n        indx0 = np.nonzero(optimalContainer_forCluster[:, 0].A == 0)[0]\n        optimalContainer_forCluster[indx1, 0] = len(multiDimensionalAverage_cluster)\n        optimalContainer_forCluster[indx0, 0] = optimalSplit_centre\n        newContainer_forCluster[np.nonzero(newContainer_forCluster[:, 0].A == optimalSplit_centre)[0], :] = optimalContainer_forCluster\n\n        multiDimensionalAverage_cluster[optimalSplit_centre] = optimal_subcentroids[0].tolist()[0] \n        multiDimensionalAverage_cluster.append(optimal_subcentroids[1].tolist()[0])\n    return np.mat(multiDimensionalAverage_cluster), newContainer_forCluster\n#the credit for the idea of this model is Jameson's","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:26.999795Z","iopub.execute_input":"2021-06-21T11:40:27.000159Z","iopub.status.idle":"2021-06-21T11:40:27.0154Z","shell.execute_reply.started":"2021-06-21T11:40:27.000127Z","shell.execute_reply":"2021-06-21T11:40:27.014404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meanOfAllPoints, newContainerForCluster = divisiveHierarchyClustering_kMeans(averageAttachedLabel, 100)\nprint(\"shape of centroids : ---->\", meanOfAllPoints.shape)\nprint('\\n')\nprint(\"centroids : ---->\", meanOfAllPoints)\nprint('\\n')\nprint('\\n')\nprint(\"shape of newContainerForCluster : ---->\", newContainerForCluster.shape)\nprint('\\n')\nprint(\"newContainerForCluster : ---->\", newContainerForCluster)\n# DISPLAY","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:28.311349Z","iopub.execute_input":"2021-06-21T11:40:28.311899Z","iopub.status.idle":"2021-06-21T11:40:32.769137Z","shell.execute_reply.started":"2021-06-21T11:40:28.311844Z","shell.execute_reply":"2021-06-21T11:40:32.767962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\ndf = px.data.iris()\nfig = px.scatter_3d(newContainerForCluster, x=0, y=1, z=0,\n              color=0, size=1, size_max=18,\n              symbol=1, opacity=0.7)\n\n# tight layout\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=0))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:37.259068Z","iopub.execute_input":"2021-06-21T11:40:37.259457Z","iopub.status.idle":"2021-06-21T11:40:40.228558Z","shell.execute_reply.started":"2021-06-21T11:40:37.259424Z","shell.execute_reply":"2021-06-21T11:40:40.227588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create submission for kaggle\nnewContainerForCluster=pd.DataFrame(newContainerForCluster) \nmeanOfAllPoints=pd.DataFrame(meanOfAllPoints) \n# #Save submission to CSV\nnewContainerForCluster.to_csv('submission.csv', index=False)  \nmeanOfAllPoints.to_csv('submission.csv', index=False)  ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T11:40:46.224111Z","iopub.execute_input":"2021-06-21T11:40:46.224505Z","iopub.status.idle":"2021-06-21T11:40:46.235715Z","shell.execute_reply.started":"2021-06-21T11:40:46.224472Z","shell.execute_reply":"2021-06-21T11:40:46.234693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}