{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Data analysis of dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport spacy\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Size of train.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Most common cleaned labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('cleaned_label').count()[['Id']].sort_values(by='Id', ascending=[False]).head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Most common noins and verbs in titles"},{"metadata":{"trusted":true},"cell_type":"code","source":"nlp = spacy.load(\"en_core_web_sm\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noun_counter = Counter()\nverb_counter = Counter()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenized = train.pub_title.apply(nlp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for t in tokenized:\n    for token in t:\n        if token.pos_ == 'NOUN':\n            noun_counter.update([token.text.lower()])\n        if token.pos_ == 'VERB':\n            verb_counter.update([token.text.lower()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noun_counter.most_common()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"verb_counter.most_common()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}