{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open('/kaggle/input/coleridgeinitiative-show-us-the-data/train/0007f880-0a9b-492d-9a58-76eb0b0e0bd7.json')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded = json.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tagged_corpus = []\nfor i in range(20):\n    sentences = loaded[i]['text']\n    words  = nltk.word_tokenize(sentences)\n    tagged_corpus.append(nltk.tag.pos_tag(words))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(tagged_corpus)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = []\nfor section in tagged_corpus:\n    sentence = []\n    for key in section:\n        if key[0] != '.':\n            sentence.append(key)\n        else:\n            sentences.append(sentence)\n            sentence = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in loaded:\n    print(x['section_title'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corp = nltk.corpus.brown.tagged_sents(categories='adventure')[:500]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.util import unique_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_list = unique_list(tag for sent in sentences for (word, tag) in sent)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"syms = unique_list(word for sent in sentences for (word, tag) in sent)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(syms))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(tag_list))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = nltk.HiddenMarkovModelTrainer(tag_list, syms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sentences)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(sentences, range(150), test_size=0.1, shuffle=True, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_test(est):\n    hmm = trainer.train_supervised(x_train, estimator=est)\n    print(\"%.2f\"%(100 * hmm.evaluate(x_val)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trigrams = list(nltk.ngrams(words,n=3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenized =  nltk.tokenize.sent_tokenize(loaded[0]['text'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.sentiment.vader import SentimentIntensityAnalyzer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sid = SentimentIntensityAnalyzer()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docs = {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_docs = {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sums = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in all_docs:\n    sums+=(all_docs[i][0]['section_title'] == 'Abstract')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_docs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sums","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in glob('/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json'):\n#     print(file)\n    f = open(file)\n    all_docs[\"doc-\"+str(i)] = json.load(f)\n    f.close()\n    i+=1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docs = {}\ni = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for doc in  all_docs:\n    sents = all_docs[doc][0]['text']\n    tokenized = nltk.tokenize.sent_tokenize(sents)\n    add = {\"compound\":0.0, \"neg\":0.0, \"neu\":0.0, \"pos\":0.0}\n    for sent in tokenized:\n        ss = sid.polarity_scores(sent)\n        for so in sorted(ss):\n            add[so]+=ss[so]\n    for s in add:\n        if len(tokenized):\n            add[s]/=len(tokenized)\n    docs[i] = add\n    i+=1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negs = []\nposis = []\nneus = []\ncompounds  = []\nfor key in docs:\n    compounds.append(docs[key]['compound'])\n    negs.append(docs[key]['neg'])\n    posis.append(docs[key]['pos'])\n    neus.append(docs[key]['neu'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.interpolate import interp1d","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def interpolate(compounds):\n    # Define x, y, and xnew to resample at.\n    x = np.array(range(len(compounds)))\n    y = np.array(compounds)\n    xnew = np.linspace(0, len(compounds) - 1, 100)\n\n    # Define interpolators.\n    f_linear = interp1d(x, y)\n    f_cubic = interp1d(x, y, kind='cubic')\n\n    # Plot.\n    plt.plot(x, y, 'o', label='data')\n    # plt.plot(xnew, f_linear(xnew), '-', label='linear')\n    plt.plot(xnew, f_cubic(xnew), '-', label='cubic')\n    plt.legend(loc='best')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sentiments for Abstract/Introduction/etc\n* Generally compounds means they cannot be properly identified and are tricky\n* Negs--> negative, Posis-->Positives\n* Neus --> Neutrals","metadata":{}},{"cell_type":"code","source":"interpolate(compounds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpolate(negs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"interpolate(neus)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpolate(posis)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mad_based_outlier(points, thresh=3.5):\n    points = np.array(points)\n    if len(points.shape) == 1:\n        points = points[:,None]\n    median = np.median(points, axis=0)\n    diff = np.sum((points - median)**2, axis=-1)\n    diff = np.sqrt(diff)\n    med_abs_deviation = np.median(diff)\n\n    modified_z_score = 0.6745 * diff / med_abs_deviation\n\n    return modified_z_score > thresh","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexes = mad_based_outlier(compounds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compounds_outliers = []\nnegatives_outliers = []\npositives_outliers = []\nneutrals_outliers = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(indexes)):\n    if indexes[i]:\n        compounds_outliers.append(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexes_2 = mad_based_outlier(negs)\nfor j in range(len(indexes_2)):\n    if indexes_2[j]:\n        negatives_outliers.append(j)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexes = mad_based_outlier(posis)\nfor j in range(len(indexes)):\n    if indexes[j]:\n        positives_outliers.append(j)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexes_2 = mad_based_outlier(neus)\nfor j in range(len(indexes_2)):\n    if indexes_2[j]:\n        neutrals_outliers.append(j)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def intersection(first, *others):\n    return set(first).intersection(*others)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}