{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/brown-clustering')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom nltk.tokenize import word_tokenize\n\nfrom brownclustering import Corpus\nfrom brownclustering import BrownClustering","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub_df=pd.read_csv('../input/publication-datasets/publication_dataset.csv')\npub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npos_sentences=[]\n\ndef get_positive_sentences(sentences):\n    sentences=eval(sentences)\n    pos_sents=sentences['pos_sents']\n    for sentence in pos_sents:\n        sentence=sentence.lower().strip()\n        words=word_tokenize(sentence)\n        words=[word.strip() for word in words]\n        words=[word for word in words if (word.isalpha() and word.isascii())]\n        pos_sentences.append(words)\n\n_=pub_df.sentences.apply(get_positive_sentences)\nprint('Number Of Positive Sentences:', len(pos_sentences))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict\nvocab=defaultdict(int)\nfor sent in pos_sentences:\n    for word in sent:\n        vocab[word]+=1\nprint('Vocab size:', len(vocab))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_freq=10\ncorpus_sentences=[]\nfor sent in pos_sentences:\n    words=[word for word in sent if vocab[word]>min_freq]\n    if len(words)>3:\n        corpus_sentences.append(words)\nprint(\"Corpus Sentences:\", len(corpus_sentences))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Corpus for the sentences with brown clusters","metadata":{}},{"cell_type":"code","source":"%%time\ncorpus=Corpus(corpus_sentences, 0.001)\nvocab=corpus.vocabulary\nunigrams=corpus.unigrams\nbigrams=corpus.bigrams\n\nprint('Vocabulary Size:',len(vocab))\nprint('Unigram Size:', len(unigrams))\nprint(\"Bigram Size:\", len(bigrams))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering = BrownClustering(corpus, 6)\nclustering.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# lets see some of the keywords for extraction","metadata":{}},{"cell_type":"code","source":"clustering.get_similar('taken')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('database')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('collected')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('sampled')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('from')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('this')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Clusters","metadata":{}},{"cell_type":"code","source":"codes=clustering.codes()\nprint(len(codes))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"creating codes of length 5","metadata":{}},{"cell_type":"code","source":"codes_df=pd.DataFrame.from_dict({\n    'word': list(codes.keys()),\n    'code_':list(codes.values())\n})\ncodes_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustering.get_similar('obtained', 30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets check the clusters of the context words of database label.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prefix=10\n\ntext1=\"the data is taken from ADNI\"\ntext2=\"This study used data from the National Education Longitudinal Study (NELS:88) to examine the effects of dual enrollment programs for high school students on college degree attainment.\"\n\nprint(text1)\nprint()\nfor word in word_tokenize(text1):\n    word=word.lower()\n    if word not in codes:\n        continue\n    print(codes[word][:prefix], \"-->\", word)\n\nprint(text2)\nprint()\nfor word in word_tokenize(text2):\n    word=word.lower()\n    if word not in codes:\n        continue\n    print(codes[word][:prefix], \"-->\", word)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"codes_df[codes_df.code_.apply(lambda x: x.startswith('0100000000'))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}