{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport seaborn as sns\nimport re\nimport nltk\nimport io\n\nimport spacy\nfrom spacy import displacy\n\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom nltk.corpus import stopwords\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample = pd.concat([pd.read_pickle(\"../input/coleridge-ner-11-train/training_df.pkl\"), pd.read_pickle(\"../input/coleridge-ner-11-train/validation_df.pkl\")]).sample(frac = 1, random_state = 42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample.to_pickle('train_sample.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm', disable=['ner', 'parser', \"tok2vec\", \"attribute_ruler\", \"lemmatizer\", \"textcat\"]) \nnlp.max_length = 3000000\nwords = set([w for w in nlp(\" \".join([i for i in train_sample.cleaned_text_training]))])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Unique words in corpus : {}\".format(len(words)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stops = stopwords.words('english')\nwords = [str(w) for w in words if w not in set(stops)]\n#words","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\n\nvocab_size = 250000\noov_token = \"<OOV>\" #out of vocabulary token\n\ntokenizer = Tokenizer(num_words = vocab_size, oov_token = oov_token)\ntokenizer.fit_on_texts(words) #train_sample.cleaned_text_training.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer_json = tokenizer.to_json()\nwith io.open('tokenizer.json', 'w', encoding='utf-8') as f:\n    f.write(json.dumps(tokenizer_json, ensure_ascii=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index\nreverse_word_index = dict([(value, key) for (key, value) in word_index.items()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_sequences = tokenizer.texts_to_sequences(train_sample.cleaned_text_training)\n#validation_sequences = tokenizer.texts_to_sequences(val_sample.cleaned_text_training)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.sequence import pad_sequences\n\nmax_length = 60\npadding_type = 'post'\ntrunc_type = 'post'\n\ntraining_padded = pad_sequences(training_sequences, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n#validation_padded = pad_sequences(validation_sequences, maxlen = max_length, padding = padding_type, truncating = trunc_type)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_labels = train_sample.BILUO_labels.values\n#validation_labels = val_sample.BILUO_labels.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer = Tokenizer(filters = ' ')\n\ntok_tr_labels = [\" \".join(i) for i in training_labels]\nlabel_tokenizer.fit_on_texts(tok_tr_labels)\ntrain_y = label_tokenizer.texts_to_sequences(tok_tr_labels)\ntrain_y = pad_sequences(train_y, maxlen = max_length, padding = padding_type, truncating = trunc_type, value = label_tokenizer.word_index['o']) - 1 #subtracting 1 to use the 'to_categorical' method\n\n#tok_val_labels = [\" \".join(i) for i in validation_labels]\n#val_y = label_tokenizer.texts_to_sequences(tok_val_labels)\n#val_y = pad_sequences(val_y, maxlen = max_length, padding = padding_type, truncating = trunc_type, value = label_tokenizer.word_index['o']) - 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer_json = label_tokenizer.to_json()\nwith io.open('label_tokenizer.json', 'w', encoding='utf-8') as f:\n    f.write(json.dumps(label_tokenizer_json, ensure_ascii=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer.word_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_pos_labels = train_sample.pos_labels.values\n#validation_pos_labels = val_sample.pos_labels.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos_tokenizer = Tokenizer(filters = ' ')\npos_tr_labels = [\" \".join(i) for i in training_pos_labels]\npos_tokenizer.fit_on_texts(pos_tr_labels)\ntrain_pos = pos_tokenizer.texts_to_sequences(pos_tr_labels)\ntrain_pos = pad_sequences(train_pos, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n\n#pos_val_labels = [\" \".join(i) for i in validation_pos_labels]\n#val_pos = pos_tokenizer.texts_to_sequences(pos_val_labels)\n#val_pos = pad_sequences(val_pos, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos_tokenizer_json = pos_tokenizer.to_json()\nwith io.open('pos_tokenizer.json', 'w', encoding='utf-8') as f:\n    f.write(json.dumps(pos_tokenizer_json, ensure_ascii=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\ntrain_y_cat = to_categorical(train_y, num_classes = len(label_tokenizer.word_index))\n#val_y_cat = to_categorical(val_y, num_classes = len(label_tokenizer.word_index))\n\n#train_pos_cat = to_categorical(train_pos, num_classes = len(pos_tokenizer.word_index))\n#val_pos_cat = to_categorical(val_pos, num_classes = len(pos_tokenizer.word_index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainlabtot = [i for j in train_sample.BILUO_labels for i in j]\n#vallabtot = [i for j in val_sample.BILUO_labels for i in j]\n\n\n\n#print(set(trainlabtot), set(vallabtot)) \n\nno_of_b = sum(np.array(trainlabtot) == 'B')\nno_of_i = sum(np.array(trainlabtot) == 'I')\nno_of_l = sum(np.array(trainlabtot) == 'L')\nno_of_u = sum(np.array(trainlabtot) == 'U')\nno_of_o = sum(np.array(trainlabtot) == 'O')\n\ntot = no_of_b + no_of_i + no_of_l + no_of_u + no_of_o\n\nprint(\"B : {} - {}%\".format(no_of_b, round(no_of_b/tot, 4)))\nprint(\"I : {} - {}%\".format(no_of_i, round(no_of_i/tot, 4)))\nprint(\"L : {} - {}%\".format(no_of_l, round(no_of_l/tot, 4)))\nprint(\"U : {} - {}%\".format(no_of_u, round(no_of_u/tot, 4)))\nprint(\"O : {} - {}%\".format(no_of_o, round(no_of_o/tot, 4)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MODEL","metadata":{}},{"cell_type":"code","source":"#!pip install git+https://www.github.com/keras-team/keras-contrib.git","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import Model, Input, Sequential\nfrom tensorflow.keras.layers import LSTM, Embedding, Dense, Concatenate\nfrom tensorflow.keras.layers import TimeDistributed, SpatialDropout1D, Bidirectional\n#from keras_contrib.layers import CRF","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DROPOUT = 0.4\n\nOUTPUT_LENGTH = len(label_tokenizer.word_index)\n\n#input for word embedding\ninput_word = Input(shape = (max_length,), name = 'input_word')#\n\n#input for pos embedding\ninput_pos = Input(shape = (max_length,), name = 'input_pos')\n\n#word embedding layer\nword_embed = Embedding(input_dim = vocab_size, output_dim = max_length, input_length = max_length, name = 'word_embedding')(input_word)\n\n#pos embedding layer\npos_embed = Embedding(input_dim = len(pos_tokenizer.word_index) + 1, output_dim = max_length, input_length = max_length, name = 'pos_embedding')(input_pos) #+1 to match the embedding \n\n#joining the two LSTMs\nconc = Concatenate()([word_embed, pos_embed])\n\n#dropout layer\nmodel = SpatialDropout1D(DROPOUT)(conc)\n\n#double BLSTM\nmodel = Bidirectional(LSTM(units = 50, return_sequences = True, recurrent_dropout = DROPOUT), name = 'word_LSTM')(model)\nmodel = Bidirectional(LSTM(units = 50, return_sequences = True, recurrent_dropout = DROPOUT, name = 'pos_LSTM'))(model)\n\n#conv layer later?\n\n#output\nout = TimeDistributed(Dense(OUTPUT_LENGTH, activation = 'softmax'))(model)\n\n#model\nmodel = Model([input_word, input_pos], out)\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\n\ndef recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n#from livelossplot.tf_keras import PlotLossesCallback","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer.word_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_class_weights(class_props, n_classes, scale = None):\n    if scale == 'log':\n        weights = np.log(1 / class_props)\n    else: \n        max_prop = np.max(class_props)\n        weights = max_prop / class_props\n    return weights\n\n\n#B : 4090 - 0.0103%\n#I : 11517 - 0.0289%\n#L : 4090 - 0.0103%\n#U : 312 - 0.0008%\n#O : 378285 - 0.9498%\n\nclass_weights = calc_class_weights([0.9498, 0.0289, 0.0103, 0.0103, 0.0008], 5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\n\n\ndef weighted_categorical_crossentropy(weights):\n    \"\"\"\n    A weighted version of keras.objectives.categorical_crossentropy\n\n    Variables:\n        weights: numpy array of shape (C,) where C is the number of classes\n\n    Usage:\n        weights = np.array([0.5,2,10]) # Class one at 0.5, class 2 twice the normal weights, class 3 10x.\n        loss = weighted_categorical_crossentropy(weights)\n        model.compile(loss=loss,optimizer='adam')\n    \"\"\"\n\n    weights = K.variable(weights)\n\n    def loss(y_true, y_pred):\n        # scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n        # clip to prevent NaN's and Inf's\n        y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n        # calc\n        loss = y_true * K.log(y_pred) * weights\n        loss = -K.sum(loss, -1)\n        return loss\n\n    return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w_categorical_crossentropy = weighted_categorical_crossentropy(class_weights)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample.iloc[0].text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainmodel = True\n\nBATCH_SIZE = 8\nEPOCHS = 15\n\nmodel.compile(optimizer =  'adam', \n              loss = w_categorical_crossentropy, # 'categorical_crossentropy', \n              metrics = ['accuracy',f1_m, precision_m, recall_m])\n\nif trainmodel:\n\n    #early_stopping = EarlyStopping(monitor = 'val_f1_m', patience = 1, verbose = 0, mode='max', restore_best_weights = True)\n\n    #callbacks = [early_stopping]\n\n    history = model.fit(\n        [training_padded, train_pos], np.array(train_y_cat),\n        #validation_data = ([validation_padded, val_pos], np.array(val_y_cat)),\n        batch_size = BATCH_SIZE,\n        epochs = EPOCHS,\n        verbose = 1,\n        #callbacks = callbacks\n\n        )\n\n    model.save('./model4.h5')\n    \nelse:\n    model.load_weights(\"../input/coleridge-ner-5-train/model2.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nplt.plot(history.history['loss'])\n#plt.plot(history.history['val_loss'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['f1_m'])\n#plt.plot(history.history['val_f1_m'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#stops = set(stopwords.words('english')).difference(['in', 'from', 'on', 'of', 's', 'at'])\n\ndef clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    text_cleaned = re.sub('[^A-Za-z0-9()-]+', ' ', str(txt)).strip()\n    \n    return text_cleaned #\" \".join([i for i in text_cleaned.split() if i not in stops])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def break_sentence(sentence, max_sentence_length, overlap):\n    \n    words = sentence.split()\n    \n    sentence_length = len(words)\n    \n    if sentence_length <= max_sentence_length:\n        return [sentence]\n    \n    else:\n        broken_sentences = []\n        \n        for p in range(0, sentence_length, max_sentence_length - overlap):\n            broken_sentences.append(\" \".join(words[p:p + max_sentence_length]))\n            \n        return broken_sentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def disambiguate_entities(entities_list):\n    \n    \"\"\"\n    This function, in case the string representing one entity contains some other entity in the list,\n    will include only the longest one.\n    \"\"\"\n    \n    entities_list = list(set(entities_list))\n    \n    final_list = []\n    \n    for e in range(len(entities_list)):\n        if entities_list[e] not in \" \".join(entities_list[:e]) + \" \".join(entities_list[e+1:]):\n            final_list.append(entities_list[e])\n            \n    return final_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tokenizer.word_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_dataset(paper_test_sentences, paper_sentences_pos, print_warn_message = False):\n    \n    #preparing data for prediction\n    tok = tokenizer.texts_to_sequences(paper_test_sentences)\n    pad = pad_sequences(tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n    \n    pos_tok = pos_tokenizer.texts_to_sequences([\" \".join(i) for i in paper_sentences_pos])\n    pos_pad = pad_sequences(pos_tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n                \n    pred = model.predict([pad, pos_pad], batch_size = BATCH_SIZE)\n        \n    pred_lab = np.argmax(pred, axis = -1)\n    \n    predtexts = []\n    \n    #mapping predictions\n    for p_idx, p in enumerate(pred_lab):\n        predictiontext = ''\n        predictionlabels = []\n        if len(set([1,2,3,4]).intersection(set(p)))>0:\n            #print(p, paper_test_sentences[p_idx])\n            for l in range(len(p)):\n                if p[l] > 0:\n                    #print(p_idx, predictiontext, tok[p_idx], len(p), len(tok[p_idx]))\n                    \n                    try:\n                        if len(predictiontext)==0:\n                            predictiontext += reverse_word_index[tok[p_idx][l]]\n                        else:\n                            if reverse_word_index[tok[p_idx][l]] not in predictiontext:\n                                predictiontext += \" {}\".format(reverse_word_index[tok[p_idx][l]])\n                        predictionlabels.append(p[l])\n                        \n                    except IndexError:\n                        \n                        if print_warn_message:\n                            print(\"Sentence: {}\".format(paper_test_sentences[p_idx]), \"The model attempted to assign a 'I' or 'B' to a padded character\")\n                        pass\n\n        else:\n            predictiontext = \"\"\n            \n            \n        if len(predictionlabels) >0:\n            \n            write = False\n            \n            \n            \n            if len(predictionlabels) == 1: #if there's only one relevant label, that should be a 'U'. Otherwise avoid producing result\n                if predictionlabels == label_tokenizer.word_index['u']-1:\n                    write = True\n                    #predtexts.append(clean_text(predictiontext))\n            \n                #if there are multiple relevant labels\n            elif label_tokenizer.word_index['l']-1 in predictionlabels or label_tokenizer.word_index['i']-1: #if there's end of sentence or middle of sentence\n                if label_tokenizer.word_index['b']-1 in predictionlabels: #there must be the beginning as well\n                    write = True\n                    \n            if write:\n                print(predictiontext, predictionlabels, paper_test_sentences[p_idx], list(zip(p, [t for t in nlp(paper_test_sentences[p_idx])])))\n                predtexts.append(clean_text(predictiontext))\n                        \n                #if label_tokenizer.word_index['b']-1 in predictionlabels: #else, if there's the beginning, it will suffice for producing the text (to be improved)\n                #predtexts.append(clean_text(predictiontext))\n        \n    return predtexts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_dataset(paper_test_sentences, paper_sentences_pos, print_warn_message = False, string_matching = False, existing_labels = []):\n    \n    #preparing data for prediction\n    tok = tokenizer.texts_to_sequences(paper_test_sentences)\n    pad = pad_sequences(tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n    \n    pos_tok = pos_tokenizer.texts_to_sequences([\" \".join(i) for i in paper_sentences_pos])\n    pos_pad = pad_sequences(pos_tok, maxlen = max_length, padding = padding_type, truncating = trunc_type)\n                \n    pred = model.predict([pad, pos_pad], batch_size = BATCH_SIZE)\n        \n    pred_lab = np.argmax(pred, axis = -1)\n    \n    predtexts = []\n    \n    #mapping predictions\n    for p_idx, p in enumerate(pred_lab):\n        predictiontext = ''\n        predictionlabels = []\n        if len(set([1,2,3,4]).intersection(set(p)))>0:\n            #print(p, paper_test_sentences[p_idx])\n            for l in range(len(p)):\n                if p[l] > 0:\n                    #print(p_idx, predictiontext, tok[p_idx], len(p), len(tok[p_idx]))\n                    \n                    try:\n                        if len(predictiontext)==0:\n                            predictiontext += reverse_word_index[tok[p_idx][l]]\n                        else:\n                            if reverse_word_index[tok[p_idx][l]] not in predictiontext:\n                                predictiontext += \" {}\".format(reverse_word_index[tok[p_idx][l]])\n                        predictionlabels.append(p[l])\n                        \n                    except IndexError:\n                        \n                        if print_warn_message:\n                            print(\"Sentence: {}\".format(paper_test_sentences[p_idx]), \"The model attempted to assign a 'I' or 'B' to a padded character\")\n                        pass\n\n        else:\n            predictiontext = \"\"\n            \n            \n        if len(predictionlabels) >0:\n            \n            write = False\n            \n            \n            \n            if len(predictionlabels) == 1: #if there's only one relevant label, that should be a 'U'. Otherwise avoid producing result\n                if predictionlabels == label_tokenizer.word_index['u']-1:\n                    write = True\n                    #predtexts.append(clean_text(predictiontext))\n            \n                #if there are multiple relevant labels\n            elif label_tokenizer.word_index['l']-1 in predictionlabels or label_tokenizer.word_index['i']-1: #if there's end of sentence or middle of sentence\n                if label_tokenizer.word_index['b']-1 in predictionlabels: #there must be the beginning as well\n                    write = True\n                    \n            if write:\n                #print(predictiontext, predictionlabels, paper_test_sentences[p_idx], list(zip(p, [t for t in nlp(paper_test_sentences[p_idx])])))\n                predtexts.append(clean_text(predictiontext))\n                        \n                #if label_tokenizer.word_index['b']-1 in predictionlabels: #else, if there's the beginning, it will suffice for producing the text (to be improved)\n                #predtexts.append(clean_text(predictiontext))\n    if string_matching:\n        for txt in paper_test_sentences:\n            for known_label in existing_labels:\n                \n                labelset = set(clean_training_text(known_label).lower().split())\n                \n                if len(labelset.intersection(set(clean_training_text(txt).lower().split()))) == len(labelset):\n                    #print(predtexts)\n                    predtexts.append(clean_text(known_label))\n        \n    return predtexts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pos_tagging_nltk(x):\n    \n    tok = word_tokenize(x)\n    \n    pos = nltk.pos_tag(tok)\n    \n    #print(x)\n    return list(zip(*pos))[1] #[nlp_feat[w].pos_ for w in range(len(nlp_feat))]\n\n\ndef pos_tagging(x):\n    \n    nlp_feat = nlp(x)\n    return [token.pos_ for token in nlp_feat]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n    \noverlap = 20 #number of overlapping words in case a sentence is broken in more sentences\n\n\ninclude_string_matching = False\n\ntest_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\ntest = pd.read_csv(test_path)\n\ntest_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n#test_sentences_dict = {}\n#test_sentences_dict['text'] = []\n#test_sentences_dict['Id'] = []\n\n\nfor paper_id in test['Id'].unique():\n    \n    paper_test_sentences = []\n    paper_sentences_pos = []\n    predtexts = []\n    \n    with open(f'{test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        #predicted_text_list = []\n        for section in paper:\n            \n            section_name = section['section_title']\n            \n            if section_name.lower() not in (): #'acknowledgements', 'acknowledgement', 'reference', 'references'):\n            \n                text = section['text']\n                #print(\"-------------------------------------------\")\n                \n                for sentence in sent_tokenize(text):\n\n                    for sub_sentence in break_sentence(sentence, max_length, overlap):\n\n                        sub_sentence = clean_training_text(sub_sentence)\n                        \n                        if len(sub_sentence)>0:\n                            #sentence_pos = pos_tagging(sub_sentence)\n\n                            paper_test_sentences.append(sub_sentence)\n                            #paper_sentences_pos.append(sentence_pos)\n                            \n    \n    for txt in nlp.pipe(paper_test_sentences, disable=['ner', 'parser', \"tok2vec\", \"attribute_ruler\", \n                                \"lemmatizer\", \"textcat\", \"attribute_ruler\", \"senter\",\n                                \"sentencizer\", \"tok2vec\"]):\n        paper_sentences_pos.append([token.pos_ for token in txt])\n        \n    #print(paper_test_sentences)\n                    \n    predtexts = predict_dataset(paper_test_sentences, paper_sentences_pos)\n    #print(predtexts)\n    \n    \n    \n    test.loc[test.Id == paper_id, 'PredictionString'] = \"|\".join(set(predtexts).difference(set([\"\"])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.PredictionString.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}