{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport seaborn as sns\nimport re\nimport nltk\n\nimport spacy\nfrom spacy import displacy\n\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom nltk.corpus import stopwords\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.text import tokenizer_from_json\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample = pd.read_pickle(\"../input/coleridge-ner-11b-train-full-dataset/train_sample.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load tokenizers\n\nwith open('../input/coleridge-ner-11b-train-full-dataset/tokenizer.json') as f:\n    data = json.load(f)\n    tokenizer = tokenizer_from_json(data)\n    \nwith open('../input/coleridge-ner-11b-train-full-dataset/label_tokenizer.json') as f:\n    data = json.load(f)\n    label_tokenizer = tokenizer_from_json(data)\n    \nwith open('../input/coleridge-ner-11b-train-full-dataset/pos_tokenizer.json') as f:\n    data = json.load(f)\n    pos_tokenizer = tokenizer_from_json(data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index\nreverse_word_index = dict([(value, key) for (key, value) in word_index.items()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size = 250000","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length = 60\npadding_type = 'post'\ntrunc_type = 'post'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MODEL","metadata":{}},{"cell_type":"code","source":"#!pip install git+https://www.github.com/keras-team/keras-contrib.git","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import Model, Input, Sequential\nfrom tensorflow.keras.layers import LSTM, Embedding, Dense, Concatenate\nfrom tensorflow.keras.layers import TimeDistributed, SpatialDropout1D, Bidirectional\n#from keras_contrib.layers import CRF","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DROPOUT = 0.4\n\nOUTPUT_LENGTH = len(label_tokenizer.word_index)\n\n#input for word embedding\ninput_word = Input(shape = (max_length,), name = 'input_word')#\n\n#input for pos embedding\ninput_pos = Input(shape = (max_length,), name = 'input_pos')\n\n#word embedding layer\nword_embed = Embedding(input_dim = vocab_size, output_dim = max_length, input_length = max_length, name = 'word_embedding')(input_word)\n\n#pos embedding layer\npos_embed = Embedding(input_dim = len(pos_tokenizer.word_index) + 1, output_dim = max_length, input_length = max_length, name = 'pos_embedding')(input_pos) #+1 to match the embedding \n\n#joining the two LSTMs\nconc = Concatenate()([word_embed, pos_embed])\n\n#dropout layer\nmodel = SpatialDropout1D(DROPOUT)(conc)\n\n#double BLSTM\nmodel = Bidirectional(LSTM(units = 50, return_sequences = True, recurrent_dropout = DROPOUT), name = 'word_LSTM')(model)\nmodel = Bidirectional(LSTM(units = 50, return_sequences = True, recurrent_dropout = DROPOUT, name = 'pos_LSTM'))(model)\n\n#conv layer later?\n\n#output\nout = TimeDistributed(Dense(OUTPUT_LENGTH, activation = 'softmax'))(model)\n\n#model\nmodel = Model([input_word, input_pos], out)\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainmodel = False\n\nBATCH_SIZE = 8\nEPOCHS = 15\n\n\n\nif trainmodel:\n    \n    model.compile(optimizer =  'adam', \n              loss = w_categorical_crossentropy, # 'categorical_crossentropy', \n              metrics = ['accuracy',f1_m, precision_m, recall_m])\n\n    #early_stopping = EarlyStopping(monitor = 'val_f1_m', patience = 1, verbose = 0, mode='max', restore_best_weights = True)\n\n    #callbacks = [early_stopping]\n\n    history = model.fit(\n        [training_padded, train_pos], np.array(train_y_cat),\n        #validation_data = ([validation_padded, val_pos], np.array(val_y_cat)),\n        batch_size = BATCH_SIZE,\n        epochs = EPOCHS,\n        verbose = 1,\n        #callbacks = callbacks\n\n        )\n\n    model.save('./model4.h5')\n    \nelse:\n    model.load_weights(\"../input/coleridge-ner-11b-train-full-dataset/model4.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#stops = set(stopwords.words('english')).difference(['in', 'from', 'on', 'of', 's', 'at'])\n\ndef clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    text_cleaned = re.sub('[^A-Za-z0-9()-]+', ' ', str(txt)).strip()\n    \n    return text_cleaned #\" \".join([i for i in text_cleaned.split() if i not in stops])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def break_sentence(sentence, max_sentence_length, overlap):\n    \n    words = sentence.split()\n    \n    sentence_length = len(words)\n    \n    if sentence_length <= max_sentence_length:\n        return [sentence]\n    \n    else:\n        broken_sentences = []\n        \n        for p in range(0, sentence_length, max_sentence_length - overlap):\n            broken_sentences.append(\" \".join(words[p:p + max_sentence_length]))\n            \n        return broken_sentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def disambiguate_entities(entities_list):\n    \n    \"\"\"\n    This function, in case the string representing one entity contains some other entity in the list,\n    will include only the longest one.\n    \"\"\"\n    \n    entities_list = list(set(entities_list))\n    \n    final_list = []\n    \n    for e in range(len(entities_list)):\n        if entities_list[e] not in \" \".join(entities_list[:e]) + \" \".join(entities_list[e+1:]):\n            final_list.append(entities_list[e])\n            \n    return final_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer.word_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_dataset(paper_test_sentences, paper_sentences_pos, print_warn_message = False, string_matching = False, existing_labels = []):\n    \n    #preparing data for prediction\n    tok = tokenizer.texts_to_sequences(paper_test_sentences)\n    pad = pad_sequences(tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n    \n    pos_tok = pos_tokenizer.texts_to_sequences([\" \".join(i) for i in paper_sentences_pos])\n    pos_pad = pad_sequences(pos_tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n                \n    pred = model.predict([pad, pos_pad], batch_size = BATCH_SIZE)\n        \n    pred_lab = np.argmax(pred, axis = -1)\n    \n    predtexts = []\n    \n    #mapping predictions\n    for p_idx, p in enumerate(pred_lab):\n        predictiontext = ''\n        predictionlabels = []\n        predictionindexes = []\n        \n        if len(set([1,2,3,4]).intersection(set(p)))>0:\n            \n            split_sentence = paper_test_sentences[p_idx].split()\n            \n            for l in range(len(p)):\n                if p[l] > 0:\n                    #print(p_idx, predictionlabels, predictiontext, tok[p_idx], len(p), len(tok[p_idx]))\n                    \n                    try:\n                        if len(predictiontext)==0:\n                            predictiontext += split_sentence[l] #reverse_word_index[tok[p_idx][l]]\n                        else:\n                            if reverse_word_index[tok[p_idx][l]] not in predictiontext:\n                                predictiontext += \" {}\".format(split_sentence[l])#reverse_word_index[tok[p_idx][l]])\n                        predictionlabels.append(p[l])\n                        predictionindexes.append(l)\n                        \n                    except IndexError:\n                        \n                        if print_warn_message:\n                            print(\"Sentence: {}\".format(paper_test_sentences[p_idx]), \"The model attempted to assign a 'I' or 'B' to a padded character\")\n                        pass\n\n        else:\n            predictiontext = \"\"\n            \n            \n        if len(predictionlabels) >0:\n            \n            write = False\n            \n            \n            if len(predictionlabels) == 1: #if there's only one relevant label, that should be a 'U'. Otherwise avoid producing result\n                if predictionlabels == label_tokenizer.word_index['u']-1:\n                    write = True\n                    #predtexts.append(clean_text(predictiontext))\n            \n            else:#if there are multiple relevant labels\n                if label_tokenizer.word_index['i']-1 in predictionlabels: #if there's end of sentence or middle of sentence\n                    try:\n                        if label_tokenizer.word_index['b']-1 in predictionlabels: #there must be the beginning as well\n                            if predictionlabels.index(label_tokenizer.word_index['b']-1) < predictionlabels.index(label_tokenizer.word_index['i']-1):\n                                write = True\n                                \n                    except ValueError:\n                        pass\n                            \n                if label_tokenizer.word_index['l']-1 in predictionlabels:\n                    try:\n                        if predictionlabels.index(label_tokenizer.word_index['l']-1) < predictionlabels.index(label_tokenizer.word_index['i']-1):\n                                write = True\n                    except ValueError:\n                        pass\n                    \n            if write:\n                #print(predictiontext, predictionlabels, paper_test_sentences[p_idx], list(zip(p, [t for t in nlp(paper_test_sentences[p_idx])])))\n                predtexts.append(clean_text(predictiontext))\n                        \n                #if label_tokenizer.word_index['b']-1 in predictionlabels: #else, if there's the beginning, it will suffice for producing the text (to be improved)\n                #predtexts.append(clean_text(predictiontext))\n    if string_matching:\n        for txt in paper_test_sentences:\n            for known_label in existing_labels:\n                \n                labelset = set(clean_training_text(known_label).lower().split())\n                \n                if len(labelset.intersection(set(clean_training_text(txt).lower().split()))) == len(labelset):\n                    #print(predtexts)\n                    predtexts.append(clean_text(known_label))\n        \n    return predtexts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pos_tagging_nltk(x):\n    \n    tok = word_tokenize(x)\n    \n    pos = nltk.pos_tag(tok)\n    \n    #print(x)\n    return list(zip(*pos))[1] #[nlp_feat[w].pos_ for w in range(len(nlp_feat))]\n\n\ndef pos_tagging(x):\n    \n    nlp_feat = nlp(x)\n    return [token.pos_ for token in nlp_feat]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm') \n    \noverlap = 20 #number of overlapping words in case a sentence is broken in more sentences\n\n\ninclude_string_matching = False\n\ntest_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\ntest = pd.read_csv(test_path)\n\ntest_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n#test_sentences_dict = {}\n#test_sentences_dict['text'] = []\n#test_sentences_dict['Id'] = []\n\n\nfor paper_id in test['Id'].unique():\n    \n    paper_test_sentences = []\n    paper_sentences_pos = []\n    predtexts = []\n    \n    with open(f'{test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        #predicted_text_list = []\n        for section in paper:\n            \n            section_name = section['section_title']\n            \n            if section_name.lower() not in (): #'acknowledgements', 'acknowledgement', 'reference', 'references'):\n            \n                text = section['text']\n                #print(\"-------------------------------------------\")\n                \n                for sentence in sent_tokenize(text):\n\n                    for sub_sentence in break_sentence(sentence, max_length, overlap):\n\n                        sub_sentence = clean_training_text(sub_sentence)\n                        \n                        if len(sub_sentence)>0:\n                            #sentence_pos = pos_tagging(sub_sentence)\n\n                            paper_test_sentences.append(sub_sentence)\n                            #paper_sentences_pos.append(sentence_pos)\n                            \n    \n    for txt in nlp.pipe(paper_test_sentences, disable=['ner', 'parser', \"tok2vec\", \"attribute_ruler\", \n                                \"lemmatizer\", \"textcat\", \"attribute_ruler\", \"senter\",\n                                \"sentencizer\", \"tok2vec\"]):\n        paper_sentences_pos.append([token.pos_ for token in txt])\n        \n    #print(paper_test_sentences)\n                    \n    predtexts = predict_dataset(paper_test_sentences, paper_sentences_pos)\n    #print(predtexts)\n    \n    \n    \n    test.loc[test.Id == paper_id, 'PredictionString'] = \"|\".join(set(predtexts).difference(set([\"\"])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.PredictionString.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}