{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T01:31:13.364742Z","iopub.execute_input":"2021-06-21T01:31:13.365193Z","iopub.status.idle":"2021-06-21T01:31:15.254025Z","shell.execute_reply.started":"2021-06-21T01:31:13.365156Z","shell.execute_reply":"2021-06-21T01:31:15.248804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\nfrom itertools import chain\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\nfrom pathlib import Path\nimport collections\nimport pickle\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport en_core_web_lg\nimport spacy\nfrom spacy.lang.en import English\nfrom spacy.util import minibatch, compounding\nfrom spacy import displacy\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.262594Z","iopub.execute_input":"2021-06-21T01:31:15.262978Z","iopub.status.idle":"2021-06-21T01:31:15.298937Z","shell.execute_reply.started":"2021-06-21T01:31:15.262945Z","shell.execute_reply":"2021-06-21T01:31:15.287191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions to Prepare Data","metadata":{}},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\n\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.30057Z","iopub.execute_input":"2021-06-21T01:31:15.301092Z","iopub.status.idle":"2021-06-21T01:31:15.306712Z","shell.execute_reply.started":"2021-06-21T01:31:15.301051Z","shell.execute_reply":"2021-06-21T01:31:15.305583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower()) ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.308906Z","iopub.execute_input":"2021-06-21T01:31:15.309385Z","iopub.status.idle":"2021-06-21T01:31:15.3205Z","shell.execute_reply.started":"2021-06-21T01:31:15.309335Z","shell.execute_reply":"2021-06-21T01:31:15.319233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.323989Z","iopub.execute_input":"2021-06-21T01:31:15.324403Z","iopub.status.idle":"2021-06-21T01:31:15.339888Z","shell.execute_reply.started":"2021-06-21T01:31:15.324367Z","shell.execute_reply":"2021-06-21T01:31:15.339039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n    \ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.34121Z","iopub.execute_input":"2021-06-21T01:31:15.34186Z","iopub.status.idle":"2021-06-21T01:31:15.357287Z","shell.execute_reply.started":"2021-06-21T01:31:15.341802Z","shell.execute_reply":"2021-06-21T01:31:15.356434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.358543Z","iopub.execute_input":"2021-06-21T01:31:15.359212Z","iopub.status.idle":"2021-06-21T01:31:15.370478Z","shell.execute_reply.started":"2021-06-21T01:31:15.35916Z","shell.execute_reply":"2021-06-21T01:31:15.369611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.373479Z","iopub.execute_input":"2021-06-21T01:31:15.37415Z","iopub.status.idle":"2021-06-21T01:31:15.388306Z","shell.execute_reply.started":"2021-06-21T01:31:15.374101Z","shell.execute_reply":"2021-06-21T01:31:15.387171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read \ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\n#review\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.390737Z","iopub.execute_input":"2021-06-21T01:31:15.391264Z","iopub.status.idle":"2021-06-21T01:31:15.486395Z","shell.execute_reply.started":"2021-06-21T01:31:15.39122Z","shell.execute_reply":"2021-06-21T01:31:15.485379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREPROCESSING","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/jagdmir/spacy-ner-model\nimport nltk\n\nDATA = []\nent_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = '../input/coleridgeinitiative-show-us-the-data/train/' + row.Id + '.json'            \n    f = open(pub)  \n    data = json.load(f)      \n    \n    balanced = False\n\n    sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(data))]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    for sentence in sentences:          \n\n        a = re.search(row.cleaned_label.lower(), sentence)\n        b = re.search(row.dataset_label.lower(), sentence)\n        c = re.search(row.dataset_title.lower(), sentence)\n        \n        if  a != None:\n            DATA.append((sentence, {\"entities\":[(a.span()[0], a.span()[1], \"DATASET\")]}))\n            ent_count = ent_count + 1\n            balanced = True\n            \n        elif b != None:\n            DATA.append((sentence, {\"entities\":[(b.span()[0], b.span()[1], \"DATASET\")]}))\n            ent_count = ent_count + 1\n            balanced = True\n            \n        elif c != None:\n            DATA.append((sentence, {\"entities\":[(c.span()[0], c.span()[1], \"DATASET\")]}))\n            ent_count = ent_count + 1\n            balanced = True\n            \n        else:\n            if balanced:\n                DATA.append((sentence, {'entities':[]}))\n                empty_count = empty_count + 1\n                balanced = False\n                \nprint('Text with entities:', ent_count)\nprint('Text without entities:', empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:31:15.487875Z","iopub.execute_input":"2021-06-21T01:31:15.488256Z","iopub.status.idle":"2021-06-21T01:49:45.086227Z","shell.execute_reply.started":"2021-06-21T01:31:15.488217Z","shell.execute_reply":"2021-06-21T01:49:45.085312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print an example of a sentence\nprint(DATA[0][0])\n\nprint('\\n')\n#print an example of the sentence's entities\nprint(DATA[0][1])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:49:45.087323Z","iopub.execute_input":"2021-06-21T01:49:45.087752Z","iopub.status.idle":"2021-06-21T01:49:45.098155Z","shell.execute_reply.started":"2021-06-21T01:49:45.08772Z","shell.execute_reply":"2021-06-21T01:49:45.097163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print an example of a sentence\nprint(DATA[10][0])\n\nprint('\\n')\n#print an example of the sentence's entities\nprint(DATA[10][1])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:49:45.099477Z","iopub.execute_input":"2021-06-21T01:49:45.100043Z","iopub.status.idle":"2021-06-21T01:49:45.116842Z","shell.execute_reply.started":"2021-06-21T01:49:45.099989Z","shell.execute_reply":"2021-06-21T01:49:45.115747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(DATA)\ntrain_df = train_df.rename({0: 'Sentence', 1: 'Entities'}, axis = 1)\n\n#review\ntrain_df.tail(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:49:45.118642Z","iopub.execute_input":"2021-06-21T01:49:45.118983Z","iopub.status.idle":"2021-06-21T01:49:45.216379Z","shell.execute_reply.started":"2021-06-21T01:49:45.118935Z","shell.execute_reply":"2021-06-21T01:49:45.21519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# spaCy Train with 'DATASET' Entity","metadata":{}},{"cell_type":"code","source":"import random\nimport spacy\nfrom spacy.util import minibatch, compounding\nfrom pathlib import Path\n\ndef train_spacy(DATA, iterations, model):\n    \n    print(f\"downloads = {model}\")\n    if model is not None:\n        print(f\"training existing model\")\n        nlp = spacy.load(model)\n        print(\"Model is Loaded '%s'\" % model)\n    else:\n        print(f\"Creating new model\")\n\n        nlp = spacy.blank('en')  #create blank language class\n\n    if 'ner' not in nlp.pipe_names:\n        ner = nlp.create_pipe('ner')\n        nlp.add_pipe(ner, last = True)\n    else:\n        ner = nlp.get_pipe('ner')\n\n    #based on template, get labels and save those for further training   \n    for _, annotations in DATA:\n        for ent in annotations.get(\"entities\"):\n            ner.add_label(ent[2])           #add 'DATASET' entities\n\n    #get names of other pipes to disable them during training\n    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']\n    with nlp.disable_pipes(*other_pipes):  # only train NER, prevent the other components from being involved\n        if model is None:\n            optimizer = nlp.begin_training()\n        else:\n            optimizer = nlp.entity.create_optimizer()\n        tags = dict()\n        for itn in range(iterations):\n            print(\"Starting iteration \" + str(itn))\n            random.shuffle(DATA)             #shuffle the examples randomly\n            losses = {}\n            batches = minibatch(DATA, size = compounding(4.0, 16.0, 1.001))\n            \n            #type 2 with mini batch\n            for batch in batches:                \n                texts, annotations = zip(*batch)                \n                golds = annotations          #zip method\n                nlp.update(texts,            #batch of texts\n                           golds,            #batch of annotations\n                           drop = 0.2,       #dropout - make it harder to memorise data\n                           losses = losses,  #hold the losses against each pipeline component\n                           sgd = optimizer)\n            print(losses)\n            \n    return nlp","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:49:45.217863Z","iopub.execute_input":"2021-06-21T01:49:45.218216Z","iopub.status.idle":"2021-06-21T01:49:45.2316Z","shell.execute_reply.started":"2021-06-21T01:49:45.218182Z","shell.execute_reply":"2021-06-21T01:49:45.230111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.datetime.now()\n\n#train\nnlp = train_spacy(DATA, 20, \"en_core_web_sm\") ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:49:45.233106Z","iopub.execute_input":"2021-06-21T01:49:45.233591Z","iopub.status.idle":"2021-06-21T03:56:06.92236Z","shell.execute_reply.started":"2021-06-21T01:49:45.233528Z","shell.execute_reply":"2021-06-21T03:56:06.921401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = datetime.datetime.now()\nelapsed = end - start\nprint('Training took a total of {}'.format(elapsed))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:06.923979Z","iopub.execute_input":"2021-06-21T03:56:06.924401Z","iopub.status.idle":"2021-06-21T03:56:06.930481Z","shell.execute_reply.started":"2021-06-21T03:56:06.924357Z","shell.execute_reply":"2021-06-21T03:56:06.929319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = 'This study used data from the National Education Longitudinal Study (NELS:88) to examine the effects of dual enrollment programs for high school students on college degree attainment.'\n\n#display sentence involving original entities\nspacy.displacy.render(nlp(text), style = \"ent\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:06.931892Z","iopub.execute_input":"2021-06-21T03:56:06.93234Z","iopub.status.idle":"2021-06-21T03:56:06.969204Z","shell.execute_reply.started":"2021-06-21T03:56:06.932259Z","shell.execute_reply":"2021-06-21T03:56:06.968387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = 'the international community came together to build the rsna international covid open radiology database ricord 1 which will be accessible to all investigator to further scientific knowledge and facilitate the development of rapid quantitative assessment tool'\n#display sentence involving original entities\nspacy.displacy.render(nlp(text), style = \"ent\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:06.972338Z","iopub.execute_input":"2021-06-21T03:56:06.972617Z","iopub.status.idle":"2021-06-21T03:56:06.993477Z","shell.execute_reply.started":"2021-06-21T03:56:06.97259Z","shell.execute_reply":"2021-06-21T03:56:06.992436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = 'the data utilized were derived from the data base our world in data 32 and had the form of cumulative infection and death data for country around the worldnin figure 2 the cumulative graph of the registered covid19infected population in france appears'\n#display sentence involving original entities\nspacy.displacy.render(nlp(text), style = \"ent\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:06.995074Z","iopub.execute_input":"2021-06-21T03:56:06.995476Z","iopub.status.idle":"2021-06-21T03:56:07.017844Z","shell.execute_reply.started":"2021-06-21T03:56:06.995433Z","shell.execute_reply":"2021-06-21T03:56:07.017104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = 'the data utilized were derived from the data base our world in data 32 and had the form of cumulative infection and death data for country around the worldnin figure 2 the cumulative graph of the registered covid19infected population in france appears'\n\ndoc = nlp(text)\n\nfor entity in doc.ents:\n    print(entity.label_, ' | ', entity.text)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:07.01895Z","iopub.execute_input":"2021-06-21T03:56:07.019433Z","iopub.status.idle":"2021-06-21T03:56:07.036726Z","shell.execute_reply.started":"2021-06-21T03:56:07.019402Z","shell.execute_reply":"2021-06-21T03:56:07.035953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize","metadata":{}},{"cell_type":"code","source":"text = 'This study used data from the National Education Longitudinal Study (NELS:88) to examine the effects of dual enrollment programs for high school students on college degree attainment.'\n\ndisplacy.render(nlp(str(text)), style='dep', jupyter = True, options = {'distance': 120})","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:07.037875Z","iopub.execute_input":"2021-06-21T03:56:07.038181Z","iopub.status.idle":"2021-06-21T03:56:07.062785Z","shell.execute_reply.started":"2021-06-21T03:56:07.038145Z","shell.execute_reply":"2021-06-21T03:56:07.062097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EVALUATION","metadata":{}},{"cell_type":"code","source":"#dictionary which will be populated with the entities and result information\nentity_evaluation = {}\n\n# helper function to udpate the entity_evaluation dictionary\ndef update_results(entity, metric):\n    if entity not in entity_evaluation:\n        entity_evaluation[entity] = {\"correct\": 0, \"total\": 0}\n    \n    entity_evaluation[entity][metric] += 1\n\n#same as before, see if entities from test set match what spaCy currently predicts\nfor data in DATA:\n    sentence = data[0]\n    entities = data[1][\"entities\"]\n\n    for entity in entities:\n        doc = nlp(sentence)\n        correct_text = sentence[entity[0]:entity[1]]\n\n        for ent in doc.ents:\n            if ent.label_ == entity[2] and ent.text == correct_text:\n                update_results(ent.label_, \"correct\")\n                break\n\n        update_results(entity[2], \"total\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:56:07.063842Z","iopub.execute_input":"2021-06-21T03:56:07.06431Z","iopub.status.idle":"2021-06-21T04:09:13.478455Z","shell.execute_reply.started":"2021-06-21T03:56:07.064262Z","shell.execute_reply":"2021-06-21T04:09:13.47739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum_total = 0\nsum_correct = 0\n\nfor entity in entity_evaluation:\n    total = entity_evaluation[entity][\"total\"]\n    correct = entity_evaluation[entity][\"correct\"]\n\n    sum_total += total\n    sum_correct += correct\n    \n    print(\"{} | {:.2f}%\".format(entity, correct / total * 100))\n\nprint()\nprint(\"Overall accuracy: {:.2f}%\".format(sum_correct / sum_total * 100))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:09:13.484445Z","iopub.execute_input":"2021-06-21T04:09:13.484811Z","iopub.status.idle":"2021-06-21T04:09:13.493718Z","shell.execute_reply.started":"2021-06-21T04:09:13.484771Z","shell.execute_reply":"2021-06-21T04:09:13.492513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nnlp.meta[\"name\"] = \"dataset_entity_extractor\"\nnlp.to_disk(\"./dataset_entity_extractor\")","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:09:13.495874Z","iopub.execute_input":"2021-06-21T04:09:13.496352Z","iopub.status.idle":"2021-06-21T04:09:13.622962Z","shell.execute_reply.started":"2021-06-21T04:09:13.496303Z","shell.execute_reply":"2021-06-21T04:09:13.621983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\npickle.dump(nlp, open( \"dataset_entity_extractor.p\", \"wb\" ))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:09:13.624634Z","iopub.execute_input":"2021-06-21T04:09:13.625135Z","iopub.status.idle":"2021-06-21T04:09:14.332194Z","shell.execute_reply.started":"2021-06-21T04:09:13.625084Z","shell.execute_reply":"2021-06-21T04:09:14.33121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"#get text\ntqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_json_pub)\n\n#clean text\ntrain['text'] = train['text'].progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:09:14.333243Z","iopub.execute_input":"2021-06-21T04:09:14.333636Z","iopub.status.idle":"2021-06-21T04:11:12.652399Z","shell.execute_reply.started":"2021-06-21T04:09:14.333606Z","shell.execute_reply":"2021-06-21T04:11:12.651626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\n#read data\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(read_json_pub, train_path = test_path))\n\n#review\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:12.653568Z","iopub.execute_input":"2021-06-21T04:11:12.653979Z","iopub.status.idle":"2021-06-21T04:11:12.741606Z","shell.execute_reply.started":"2021-06-21T04:11:12.653947Z","shell.execute_reply":"2021-06-21T04:11:12.740421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:12.743198Z","iopub.execute_input":"2021-06-21T04:11:12.743605Z","iopub.status.idle":"2021-06-21T04:11:12.760474Z","shell.execute_reply.started":"2021-06-21T04:11:12.743559Z","shell.execute_reply":"2021-06-21T04:11:12.759407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_matching = True\nspacy_prediction = True\n\nid_list = []\nlabels_list = []\n\nfor index, row in tqdm(sample_submission.iterrows()):\n\n    sample_text = row['text']\n\n    row_id = row['Id']\n    \n    #takes only the rows where train file is identical to a test file\n    temp_df = train[train['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    #literal_matching \n    if literal_matching:\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():    \n                cleaned_labels.append(clean_text(known_label))\n\n        print('cleaned label:', set(cleaned_labels))\n        \n    #spaCy NER \n    if spacy_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n        #for each entity, use our updated model to make a prediction on the sentence\n        for sentence in sentences:\n            \n            doc = nlp(sentence)\n\n            for ent in doc.ents:\n                spacy_labels = ent.text\n                \n        print('spacy label:', spacy_labels)\n\n    cleaned_labels.append(spacy_labels)\n        \n    cleaned_labels = set(cleaned_labels)\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]    \n    labels_list.append('|'.join(cleaned_labels))\n    print('label list:', labels_list)   \n    id_list.append(row_id)\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:12.761861Z","iopub.execute_input":"2021-06-21T04:11:12.762187Z","iopub.status.idle":"2021-06-21T04:11:36.260519Z","shell.execute_reply.started":"2021-06-21T04:11:12.762154Z","shell.execute_reply":"2021-06-21T04:11:36.259471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\nsample_submission['PredictionString'] = labels_list\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:36.261913Z","iopub.execute_input":"2021-06-21T04:11:36.26228Z","iopub.status.idle":"2021-06-21T04:11:36.276788Z","shell.execute_reply.started":"2021-06-21T04:11:36.262246Z","shell.execute_reply":"2021-06-21T04:11:36.275552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission['PredictionString'][0])\nprint('\\n')\nprint(sample_submission['PredictionString'][1])\nprint('\\n')\nprint(sample_submission['PredictionString'][2])\nprint('\\n')\nprint(sample_submission['PredictionString'][3])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:36.278362Z","iopub.execute_input":"2021-06-21T04:11:36.278686Z","iopub.status.idle":"2021-06-21T04:11:36.296003Z","shell.execute_reply.started":"2021-06-21T04:11:36.278653Z","shell.execute_reply":"2021-06-21T04:11:36.294567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop 'text'\nsample_submission.drop(columns = 'text', axis = 1, inplace = True)\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:11:36.297465Z","iopub.execute_input":"2021-06-21T04:11:36.297797Z","iopub.status.idle":"2021-06-21T04:11:36.313111Z","shell.execute_reply.started":"2021-06-21T04:11:36.297765Z","shell.execute_reply":"2021-06-21T04:11:36.312086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsample_submission.to_csv('submission.csv', index = False)\n\n#check\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:41:54.215717Z","iopub.execute_input":"2021-06-21T04:41:54.21617Z","iopub.status.idle":"2021-06-21T04:41:54.233222Z","shell.execute_reply.started":"2021-06-21T04:41:54.216132Z","shell.execute_reply":"2021-06-21T04:41:54.231864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}