{"cells":[{"metadata":{},"cell_type":"markdown","source":"# [Show Us The Data] Topic Modelling with LDA\n\n## Reference\n\n- [LDA and T-SNE Interactive Visualization](https://www.kaggle.com/ykhorramz/lda-and-t-sne-interactive-visualization/data?select=Papers.csv)\n- [Merge multiple JSON files to a DATAFRAME](https://www.kaggle.com/hamditarek/merge-multiple-json-files-to-a-dataframe?select=df_train.csv)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pickle as pk\nfrom scipy import sparse as sp\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { max-width:100% !important; }</style>\"))\ndisplay(HTML(\"<style>.output_result { max-width:100% !important; }</style>\"))\ndisplay(HTML(\"<style>.output_area { max-width:100% !important; }</style>\"))\ndisplay(HTML(\"<style>.input_area { max-width:100% !important; }</style>\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%pylab inline\ndf = pd.read_csv('../input/merge-multiple-json-files-to-a-dataframe/df_train.csv').sample(1000)\ndocs = array(df['text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk.tokenize import RegexpTokenizer\n\ndef docs_preprocessor(docs):\n    tokenizer = RegexpTokenizer(r'\\w+')\n    for idx in range(len(docs)):\n        docs[idx] = str(docs[idx]).lower()  # Convert to lowercase.\n        docs[idx] = tokenizer.tokenize(docs[idx])  # Split into words.\n\n    # Remove numbers, but not words that contain numbers.\n    docs = [[token for token in doc if not token.isdigit()] for doc in docs]\n    \n    # Remove words that are only one character.\n    docs = [[token for token in doc if len(token) > 3] for doc in docs]\n    \n    # Lemmatize all words in documents.\n    lemmatizer = WordNetLemmatizer()\n    docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]\n  \n    return docs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"docs = docs_preprocessor(docs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from gensim.models import Phrases\n# Add bigrams and trigrams to docs (only ones that appear 10 times or more).\nbigram = Phrases(docs, min_count=10)\ntrigram = Phrases(bigram[docs])\n\nfor idx in range(len(docs)):\n    for token in bigram[docs[idx]]:\n        if '_' in token:\n            # Token is a bigram, add to document.\n            docs[idx].append(token)\n    for token in trigram[docs[idx]]:\n        if '_' in token:\n            # Token is a bigram, add to document.\n            docs[idx].append(token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from gensim.corpora import Dictionary\n\n# Create a dictionary representation of the documents.\ndictionary = Dictionary(docs)\nprint('Number of unique words in initital documents:', len(dictionary))\n\n# Filter out words that occur less than 10 documents, or more than 20% of the documents.\ndictionary.filter_extremes(no_below=10, no_above=0.2)\nprint('Number of unique words after removing rare and common words:', len(dictionary))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"corpus = [dictionary.doc2bow(doc) for doc in docs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"print('Number of unique tokens: %d' % len(dictionary))\nprint('Number of documents: %d' % len(corpus))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from gensim.models import LdaModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Set training parameters.\nnum_topics = 10\nchunksize = 500 # size of the doc looked at every pass\npasses = 20 # number of passes through documents\niterations = 100\neval_every = 1  # Don't evaluate model perplexity, takes too much time.\n\n# Make a index to word dictionary.\ntemp = dictionary[0]  # This is only to \"load\" the dictionary.\nid2word = dictionary.id2token\n\n%time model = LdaModel(corpus=corpus, id2word=id2word, chunksize=chunksize, \\\n                       alpha='auto', eta='auto', \\\n                       iterations=iterations, num_topics=num_topics, \\\n                       passes=passes, eval_every=eval_every)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pyLDAvis.gensim\nimport warnings\n\npyLDAvis.enable_notebook()\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pyLDAvis.gensim.prepare(model, corpus, dictionary)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}