{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport time\nimport pickle\nimport string\nimport json\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\n\n\nfrom torch.nn.functional import softmax\nfrom collections import defaultdict, Counter\nfrom sklearn.metrics import confusion_matrix, f1_score\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom spacy.lang.en.stop_words import STOP_WORDS\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T06:47:26.145001Z","iopub.execute_input":"2021-06-18T06:47:26.14535Z","iopub.status.idle":"2021-06-18T06:47:27.322086Z","shell.execute_reply.started":"2021-06-18T06:47:26.145322Z","shell.execute_reply":"2021-06-18T06:47:27.320852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device( 'cuda' if  torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.323891Z","iopub.execute_input":"2021-06-18T06:47:27.32428Z","iopub.status.idle":"2021-06-18T06:47:27.332816Z","shell.execute_reply.started":"2021-06-18T06:47:27.324248Z","shell.execute_reply":"2021-06-18T06:47:27.331752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config={\n    'label_batch_size': 64,\n    'unlabel_batch_size': 64,\n    'test_batch_size': 150,\n    'max_seq_len': 100,\n    'glove_dim': 100,\n    'num_labels': 3,\n    'ner_format': \"BIO\",\n    'grad_clip_val': 1,\n    'base_lr': 1e-5,\n    'max_lr': 1e-4,\n    'T': 0.6,\n    'weight_decay': 1e-5,\n    'num_iterations': 50,\n    'print_every': 10,\n    'eval_every':10,\n    'overlap_len': 20,\n    'glove_path': '../input/glove6b/glove.6B.100d.txt',\n    'unlabeled_datapath': '../input/dataset/train_data.pkl',\n    'train_folder': \"../input/coleridgeinitiative-show-us-the-data/train\",\n    'test_folder': '../input/coleridgeinitiative-show-us-the-data/test',\n    'ss_folder': '../input/coleridge-semisuperviseddata',\n    'model_path': '../input/pretrain1/best_fscore_model.pt',\n    'test_dir': '../input/coleridgeinitiative-show-us-the-data/test'\n}\n\nmodel_params={\n    \"pre_embedd_dim\": 100,\n    'word_shape_size': 7,\n    \"word_shape_embedd_dim\": 20,\n    \"hdim\": 128,\n    \"proj_dim\": 512,\n    \"out_dim\": 3,\n    'max_seq_len': 100,\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.334817Z","iopub.execute_input":"2021-06-18T06:47:27.335185Z","iopub.status.idle":"2021-06-18T06:47:27.346607Z","shell.execute_reply.started":"2021-06-18T06:47:27.335154Z","shell.execute_reply":"2021-06-18T06:47:27.345508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.348482Z","iopub.execute_input":"2021-06-18T06:47:27.348873Z","iopub.status.idle":"2021-06-18T06:47:27.367108Z","shell.execute_reply.started":"2021-06-18T06:47:27.348838Z","shell.execute_reply":"2021-06-18T06:47:27.366196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cleaning With Jaccard is taken from https://www.kaggle.com/tungmphung/coleridge-matching-bert-ner","metadata":{}},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"exclude_entities=['table', 'fig', 'provide','data','mri', 'result']\n\n\ndef get_word_properties(word):\n    prop={}\n    \n    prop['word']=word\n    prop['is_alpha']=False\n    prop['is_title']=False\n    prop['is_upper']=False\n    prop['is_lower']=False\n    prop['has_upper']=False\n    prop['is_number']=False\n    prop['is_stopword']=False\n    prop['is_punct'] = False\n    prop['alpha_1']=False\n    \n    \n    if word.isalpha():\n        prop['is_alpha']=True\n    if word[0].isalpha():\n        prop['alpha_1']=True\n    if word.islower():\n        prop['is_lower']=True\n    if word.isupper():\n        prop['is_upper']=True\n        prop['has_upper']=True\n    if word.lower() in STOP_WORDS:\n        prop['is_stopword']=True\n    if word.istitle():\n        prop['is_title']=True\n        prop['has_upper']=True\n    if word.isnumeric():\n        prop['is_number']=True\n        \n    if word in string.punctuation:\n        prop['is_punct']=True\n    \n    if (prop['is_alpha']) and (not prop['has_upper']) and (not prop['is_lower']):\n        for ch in word:\n            if ch.isupper():\n                prop['has_upper']=True\n    return prop\n\n\ndef get_candidate_index(i, word_props, word_len, candidates):\n    word=word_props[i]['word'].lower()\n    if (candidates[i-1] == 0) and  (word_props[i]['has_upper']) and (not word_props[i]['is_stopword']):\n        return 1\n    \n    if candidates[i-1]==1 or candidates[i-1]==2:\n        if word_props[i]['is_punct'] and word in ['(', ')', '-']:\n            return 2\n        if word_props[i]['is_punct']:\n            return 0\n        if word_props[i]['has_upper']:\n            return 2\n        if word_props[i]['is_lower'] and (i+1 < word_len) and (word_props[i+1]['is_lower']):\n            return 0\n        if (word_props[i]['is_lower'] and (i+1 < word_len) and \n                (word_props[i+1]['has_upper']) and word in ['in', 'for', 'of', 'the', 'and']):\n            return 2\n    return 0\n    \ndef get_candidate_entities(sentence):\n    words=word_tokenize(sentence)\n    words=[word.strip() for word in words]\n    words=[word for word in words if len(word)>0]\n    if words[0].isnumeric():\n        words=words[1:]\n    words_len=len(words)\n    word_props=[]\n    candidates=[0]*words_len\n    cwords=[]\n    if words_len <= 5:\n        return (candidates, cwords)\n    \n    for word in words:\n        prop=get_word_properties(word)\n        word_props.append(prop)\n    \n    if (not word_props[0]['is_stopword']) and (word_props[0]['has_upper'] and (not word_props[1]['word']==',')):\n        candidates[0]=1\n    \n    for i in range(1, words_len):\n        prop=word_props[i]\n        word=prop['word']\n        candidates[i]=get_candidate_index(i, word_props, words_len, candidates)\n    \n    #Removing the first word sequence as candidate words.\n    if candidates[0]==1:\n        candidates[0]=0\n        for i in range(1, words_len):\n            if candidates[i]==1:\n                break\n            candidates[i]=0\n    s=-1; e=-1\n    for i in range(words_len):\n        if candidates[i]==1 and s==-1:\n            s=i\n            e=i\n        elif candidates[i]==1 and s!=-1:\n            cwords.append(' '.join(words[s: e+1]))\n            s=i\n        elif candidates[i]==2:\n            e=i\n        elif s!=-1 and candidates[i]==0:\n            cwords.append(' '.join(words[s: e+1]))\n            s=-1\n            e=-1\n    cwords=[word for word in cwords if (len(word)>2) and (not word[0].isnumeric())]\n    return (candidates, cwords)\n\n\ndef get_abbrevation(sentence_words, sentence_len, i):\n    if i==sentence_len or sentence_words[i]!='(':\n        return ''\n    else:\n        j=i\n        while j<sentence_len:\n            if sentence_words[j] ==')':\n                break\n            j+=1\n        abbr=sentence_words[i+1: j]\n        abbr=[word.strip() for word in abbr]\n        abbr=[word for word in abbr if len(word)!=0]\n        if len(abbr) ==1 and abbr[0] == 1:\n            return ''\n        if not abbr[0][0].isupper():\n            return ''\n    return ''.join(sentence_words[i: j+1])\n        \n\ndef get_abbrevated_labels(sentence, all_labels):\n    sentence_words=word_tokenize(sentence)\n    sentence_words=[w for w in sentence_words if w!='']\n    sentence_len=len(sentence_words)\n    abbr_labels=[]\n    for label in all_labels:\n        label_words=word_tokenize(label)\n        for i, sword in enumerate(sentence_words):\n            flag=True\n            for j, lword in enumerate(label_words):\n                if sentence_words[i+j] != lword:\n                    flag=False\n                    break\n            if flag:\n                abbr=get_abbrevation(sentence_words, sentence_len, i+len(label_words))\n                abbr_labels.append(label+\" \"+abbr)\n                break\n    return abbr_labels\n\n\n\ndef get_new_entities(candidates, all_labels, sentence):\n    new_ents=[]\n    if len(all_labels)==0:\n        return []\n    sent_words=word_tokenize(sentence)\n    words_len=len(sent_words)\n    entity_markers=[0]*words_len\n    ent_id=1\n    \n    try:\n        for label in all_labels:\n            lwords=word_tokenize(label)\n            for i in range(words_len):\n                flag=True\n                for j in range(len(lwords)):\n                    if lwords[j] != sent_words[i+j]:\n                        flag=False\n                        break\n                if flag:\n                    for k in range(i, i+len(lwords)):\n                        entity_markers[k]=ent_id\n                    ent_id+=1\n        for i, em in enumerate(entity_markers):\n            if (em==0):\n                continue\n            tol=1\n            if (i-1)>=0 and entity_markers[i-1]==0 and sent_words[i-1]==',':\n                s=-1;e=-1\n                for j in range(i-2, -1, -1):\n                    if tol==0 or entity_markers[j]!=0:\n                        s=-1;e=-1;\n                        break\n                    if e==-1 and candidates[j]==0 and sent_words[j]!=')':\n                        tol-=1\n\n                    if s!=-1 and candidates[j]==1:\n                        s=j\n                        break\n\n                    if e==-1 and candidates[j]!=0:\n                        e=j;s=j;tol=2;\n                        if candidates[j]==1:\n                            break\n\n                    if candidates[j]==2:\n                        s=j\n                if s!=-1:\n                    new_ents.append(sent_words[s:e+1])\n\n\n            tol=1\n            if (i+1)<words_len and entity_markers[i+1]==0 and (sent_words[i+1]==',' or sent_words[i+1]=='and'):\n                s=-1; e=-1\n                for j in range(i+2, words_len):\n                    if tol==0 or entity_markers[j]!=0:\n                        s=-1; e=-1\n                        break\n                    if s==-1 and candidates[j]==0 and sent_words[j]!='(':\n                        tol-=1\n                    if s==-1 and candidates[j]!=0:\n                        s=j; e=j; tol=2\n                    if s!=-1 and (candidates[j]==1 or candidates[j]==0):\n                        e=j-1\n                        break\n                    else:\n                        e=j\n                if s!=-1:\n                    new_ents.append(sent_words[s:e+1])\n    except:\n        pass\n    \n    new_ents_final=[]\n    for ent in new_ents:\n        flag=True\n        for f in exclude_entities:\n            if f in ent:\n                flag=False\n                break\n        if flag:\n            new_ents_final.append(ent)\n    return new_ents_final\n\ndef get_datalabels(pub_id):\n    data=get_publications_data(pub_id, config['test_dir'])\n    all_labels=[]\n    new_entities=[]\n    for section in data:\n        text=section['text']\n        sentences=sent_tokenize(text)\n        for sentence in sentences:\n            sentence_labels=[]\n            for label in database:\n                if label in sentence:\n                    sentence_labels.append(label)\n            if len(sentence_labels) != 0:\n                abrevation_labels=get_abbrevated_labels(sentence, sentence_labels)\n                all_labels+=sentence_labels\n                all_labels+=abrevation_labels\n                \n                #Taken from https://www.kaggle.com/tungmphung/pytorch-bert-for-named-entity-recognition/comments\n                if any( [word in sentence.lower() for word in ['data', 'study']] ): \n                    (candidates, cwords)=get_candidate_entities(sentence)\n                    new_entities+=get_new_entities(candidates, all_labels, sentence)\n    return (all_labels, new_entities)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.368462Z","iopub.execute_input":"2021-06-18T06:47:27.368969Z","iopub.status.idle":"2021-06-18T06:47:27.421754Z","shell.execute_reply.started":"2021-06-18T06:47:27.368916Z","shell.execute_reply":"2021-06-18T06:47:27.420146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_annotation(annots, s, l):\n    for i in range(s, s+l):\n        if annots[i] == 'I':\n            continue\n        if i == s:\n            annots[i]='B'\n        else:\n            annots[i]='I'\n\ndef get_annotated_data(data):\n    sentence=data['sentence']\n    labels=data['labels'] if data.get('labels', None) else []\n    words=word_tokenize(sentence)\n    annots=['O']*len(words)\n    labels=[word_tokenize(label) for label in labels]\n    \n    for i, word in enumerate(words):\n        for label in labels:\n            if words[i:i+len(label)] == label:\n                update_annotation(annots, i, len(label))\n    return (words, annots)\n\ndef read_file(filepath):\n    with open(filepath) as file:\n        data=file.read()\n    return data\n\ndef read_json_file(filepath):\n    data=json.loads(read_file(filepath))\n    return data\ndef read_pickle(filepath):\n    with open(filepath, 'rb') as file:\n        data=pickle.load(file)\n    return data\n\ndef get_word_shape(word):\n    if len(word)==0:\n        return 0\n    ch=word[0]\n    if ch.isupper():\n        return 1\n    elif ch.islower():\n        return 2\n    elif ch.isnumeric():\n        return 3\n    elif ch == ',':\n        return 4\n    elif ch == '(':\n        return 5\n    elif ch==')':\n        return 6\n    return 0\n\n\ndef get_publications_data(pub_id, dirname):\n    pub_filename=dirname+'/{}'.format(pub_id)\n    with open(pub_filename) as file:\n        data=json.load(file)\n    return data\n\ndef get_short_sentences(sentence):\n    sentence_words=word_tokenize(sentence)\n    short_sentences=[]\n    if len(sentence_words) < config['max_seq_len']:\n        return [sentence]\n    \n    for i in range(0, len(sentence_words), config['max_seq_len']-config['overlap_len']):\n        if len(sentence_words[i:i+config['max_seq_len']]) <= 10:\n            continue\n        short_sentences.append( ' '.join(sentence_words[i:i+config['max_seq_len']] ))\n    return short_sentences\n\ndef is_eligible_sentence(sentence_words):\n    lower_cnt=0\n    for word in sentence_words:\n        if word.islower():\n            lower_cnt+=1\n    if (lower_cnt<=5) or (len(sentence_words)>=300):\n        return False\n    return True\n\ndef get_sentences(pub_id, text):\n    sentences=sent_tokenize(text)\n    short_sentences=[]\n    infer_sentences=[]\n    \n    for sentence in sentences:\n        short_sentences+=get_short_sentences(sentence)\n    for sentence in short_sentences:\n        sentence_words=word_tokenize(sentence)\n        if (not is_eligible_sentence(sentence_words)):\n            continue\n        if any([word in sentence.lower() for word in ['study', 'data']]):\n            infer_sentences.append(sentence_words)\n    return infer_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.423916Z","iopub.execute_input":"2021-06-18T06:47:27.424617Z","iopub.status.idle":"2021-06-18T06:47:27.448043Z","shell.execute_reply.started":"2021-06-18T06:47:27.424565Z","shell.execute_reply":"2021-06-18T06:47:27.446615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing Data","metadata":{}},{"cell_type":"code","source":"class PreprocessData:\n    def __init__(self, config):\n        self.train_folder=config['train_folder']\n        self.ss_folder=config['ss_folder']\n        self.glove_path=config['glove_path']\n        self.unlabled_path=config['unlabeled_datapath']\n    \n    def download_glove(self):\n        glove_embeddings={}\n        with open(self.glove_path) as file:\n            for line in file:\n                line=line.split()\n                word=line[0]\n                v=np.array(line[1:]).astype(np.float)\n                glove_embeddings[word]=v\n        return glove_embeddings\n\n    def get_labeled_data(self):\n        pos_data=read_json_file(os.path.join(config['ss_folder'], 'annotation.txt'))\n        neg_data=read_json_file(os.path.join(config['ss_folder'], 'negative_sentences_corrected.txt'))\n        return (pos_data, neg_data)\n    \n    def get_data(self):\n        (pos_data, neg_data)=self.get_labeled_data()\n        unlabeled_data=read_pickle(self.unlabled_path)\n        for data in pos_data:\n            pub_id=data.get('pub_id', None)\n            unlabeled_data.pop(pub_id) if unlabeled_data.get(pub_id, None) else ''\n        for data in neg_data:\n            pub_id=data.get('pub_id', None)\n            unlabeled_data.pop(pub_id) if unlabeled_data.get(pub_id, None) else ''\n        \n        unlabeled_data=list(unlabeled_data.values())\n        pos_data=[get_annotated_data(data) for data in pos_data]\n        neg_data=[get_annotated_data(data) for data in neg_data]\n        \n        glove_embeddings=self.download_glove()\n        labeled_data={\n            'pos_data': pos_data,\n            'neg_data': neg_data\n        }\n        return (glove_embeddings, labeled_data, unlabeled_data)\n\n(glove_embeddings, labeled_data, unlabeled_data)=PreprocessData(config).get_data()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:27.450773Z","iopub.execute_input":"2021-06-18T06:47:27.451421Z","iopub.status.idle":"2021-06-18T06:48:17.353666Z","shell.execute_reply.started":"2021-06-18T06:47:27.451371Z","shell.execute_reply":"2021-06-18T06:48:17.352969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset & DataIterators","metadata":{}},{"cell_type":"code","source":"class LabeledIterator:\n    def __init__(self,config,glove_embeddings, labeled_data):\n        self.batch_size=config['label_batch_size']\n        self.glove_dim=config['glove_dim']\n        self.num_labels=config['num_labels']\n        self.max_seq_len=config['max_seq_len']\n        self.glove_embeddings=glove_embeddings\n        self.labeled_data=labeled_data\n        \n        \n        pos_data=labeled_data['pos_data']\n        neg_data=labeled_data['neg_data']\n        \n        pos_ids=list(range(len(pos_data)))\n        neg_ids=list(range(len(neg_data)))\n        \n        pos_train_ids=pos_ids[:-1]\n        neg_train_ids=neg_ids[:-1]\n        \n        pos_val_ids=pos_ids[-1:]\n        neg_val_ids=neg_ids[-1:]\n        \n        self.train_data=([data for i, data in enumerate(pos_data) if i in pos_train_ids]+\n                         [data for i, data in enumerate(neg_data) if i in neg_train_ids])\n        \n        self.val_data  =([data for i, data in enumerate(pos_data) if i in pos_val_ids]+\n                         [data for i, data in enumerate(neg_data) if i in neg_val_ids])\n        \n        \n        self.num_train_records=len(self.train_data)\n        self.num_val_records=len(self.val_data)\n    \n    def make_shuffle_data(self, data, shuffle):\n        if shuffle:\n            np.random.shuffle(data)\n            \n    def get_data_by_mode(self, mode, shuffle):\n        if mode=='val':\n            self.make_shuffle_data(self.val_data, shuffle)\n            num_records=self.num_val_records\n            data=self.val_data\n        elif mode == 'train':\n            self.make_shuffle_data(self.train_data, shuffle)\n            num_records=self.num_train_records\n            data=self.train_data\n        return num_records, data\n   \n    def convert_annotation_to_label(self, annot):\n        if annot == 'B':\n            return 1\n        elif annot =='I':\n            return 2\n        return 0\n    \n    def convert_rawdata_to_tensors(self, mbs):\n        mb_size=len(mbs)\n        X=torch.zeros((mb_size, self.max_seq_len, self.glove_dim), dtype=torch.float32)\n        X_embedd=torch.zeros( (mb_size, self.max_seq_len), dtype=torch.long)\n        y_bios=torch.full((mb_size, self.max_seq_len), 3, dtype=torch.long)\n        y_ents=torch.zeros((mb_size, self.max_seq_len), dtype=torch.float32)\n        slens=torch.zeros(mb_size, dtype=torch.long)\n        \n        for i in range(mb_size):\n            (words, annots)=mbs[i]\n            words_len=min(len(words), self.max_seq_len)\n            slens[i]=words_len\n            for j in range(words_len):\n                wshape=get_word_shape(words[j])\n                word=words[j].lower()\n                \n                X_embedd[i][j]=wshape\n                y_bios[i][j]=self.convert_annotation_to_label(annots[j])\n                if y_bios[i][j] == 1 or y_bios[i][j]==2:\n                    y_ents[i][j]=1\n                if word in self.glove_embeddings:\n                    X[i][j]=torch.tensor(self.glove_embeddings[word], dtype=torch.float32)\n                    \n        return (X, X_embedd, y_ents, y_bios, slens)\n    def get_raw_minibatch(self, mode='val', shuffle=False):\n        num_records, data=self.get_data_by_mode(mode, shuffle)\n        for i in range(0, num_records, self.batch_size):\n            yield data[i:i+self.batch_size]\n            \n    def get_minibatch(self, mode='val', shuffle=False):\n        for mbs in self.get_raw_minibatch(mode, shuffle):\n            mbs=self.convert_rawdata_to_tensors(mbs)\n            yield mbs\n    def get_infinite_minibatch(self, mode='val', shuffle=False):\n        while True:\n            for mbs in self.get_minibatch(mode, shuffle):\n                yield mbs\n    def __iter__(self):\n        while True:\n            for mbs in self.get_raw_minibatch('train', True):\n                mbs=self.convert_rawdata_to_tensors(mbs)\n                yield mbs","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.35575Z","iopub.execute_input":"2021-06-18T06:48:17.356215Z","iopub.status.idle":"2021-06-18T06:48:17.380805Z","shell.execute_reply.started":"2021-06-18T06:48:17.35617Z","shell.execute_reply":"2021-06-18T06:48:17.380065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, max_seq_len, glove_dim):\n        self.df=df\n        self.max_seq_len=max_seq_len\n        self.glove_dim=glove_dim\n        \n    def __getitem__(self, idx):\n        row=self.df.iloc[idx]\n        sentence_words=row.sentence_words\n        \n        X=torch.zeros((self.max_seq_len, self.glove_dim))\n        X_embedd=torch.zeros(self.max_seq_len, dtype=torch.long)\n        slen=torch.tensor( min(self.max_seq_len, len(sentence_words)), dtype=torch.long)\n        for i, word in enumerate(sentence_words):\n            if i>=self.max_seq_len:\n                break\n            wshape=get_word_shape(sentence_words[i])\n            word=word.lower()\n            X_embedd[i]=wshape\n            if word in glove_embeddings:\n                X[i]=torch.tensor(glove_embeddings[word], dtype=torch.float32)\n        return (slen, X, X_embedd)\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.382371Z","iopub.execute_input":"2021-06-18T06:48:17.382842Z","iopub.status.idle":"2021-06-18T06:48:17.396779Z","shell.execute_reply.started":"2021-06-18T06:48:17.382786Z","shell.execute_reply":"2021-06-18T06:48:17.395551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class PrimaryHead(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x\n    \nclass AuxilaryHead(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], 1)\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x    \n\nclass Model(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.params=params\n        self.embedd_layer=nn.Embedding(params['word_shape_size'], params['word_shape_embedd_dim'],\n                                       max_norm=1, padding_idx=0)\n        self.gru=nn.GRU(params['pre_embedd_dim'] + params['word_shape_embedd_dim'],\n                        params['hdim'], num_layers=2,\n                        bidirectional=True, dropout=0.3, batch_first=True)\n        \n        self.primary_head=PrimaryHead(params)\n        self.aux_head=AuxilaryHead(params)\n    \n    def forward(self, x, x_embedd):\n        batch_size=x.shape[0]\n        seq_len=x.shape[1]\n        x_embedd=self.embedd_layer(x_embedd)\n        (h_n, _)=self.gru(torch.cat([x, x_embedd], dim=-1))\n        h_n=h_n.view(batch_size, seq_len, 2, self.params['hdim'] )\n        h1=h_n[:, :, 0, :]\n        h2=h_n[:, :, 1, :]\n        \n        h=torch.cat([h1, h2], dim=-1)\n        y_ent=self.aux_head(h)\n        y_bios=self.primary_head(h)\n        return (y_ent, y_bios)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.398483Z","iopub.execute_input":"2021-06-18T06:48:17.398881Z","iopub.status.idle":"2021-06-18T06:48:17.420429Z","shell.execute_reply.started":"2021-06-18T06:48:17.398841Z","shell.execute_reply":"2021-06-18T06:48:17.4195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(model):\n    H=0.0;num_records=0;\n    true_ent_labels=[];pred_ent_labels=[];\n    true_bio_labels=[];pred_bio_labels=[];\n    \n    model.eval()\n    labeled_iterator=LabeledIterator(config, glove_embeddings, labeled_data)\n    for mbs in labeled_iterator.get_minibatch(mode='train', shuffle=False):\n        (X, X_embedd, y_ents, y_bios, _)=mbs\n        with torch.no_grad():\n            (yhat_ent, yhat_bios)=model(X, X_embedd)\n            yhat_ent=(yhat_ent>0).to(int)\n            yhat_bios=yhat_bios.argmax(dim=-1)\n            num_records+=1\n        \n        batch_size=X.shape[0]\n        y_ents=y_ents.view(-1)\n        yhat_ent=yhat_ent.view(-1)\n        \n        y_bios=y_bios.view(-1)\n        yhat_bios=yhat_bios.view(-1)\n        \n        \n        yhat_bios=yhat_bios[y_bios!=3]\n        yhat_ent=yhat_ent[y_bios!=3]\n        y_ents=y_ents[y_bios!=3]\n        y_bios=y_bios[y_bios!=3]\n        \n        true_ent_labels+=list(y_ents.numpy())\n        pred_ent_labels+=list(yhat_ent.numpy())\n        \n        true_bio_labels+=list(y_bios.numpy())\n        pred_bio_labels+=list(yhat_bios.numpy())\n    \n    cm1=confusion_matrix(true_ent_labels, pred_ent_labels)\n    cm2=confusion_matrix(true_bio_labels, pred_bio_labels)\n    \n    micro_fscore_ent=f1_score(true_ent_labels, pred_ent_labels, average='micro')\n    micro_fscore_bio=f1_score(true_bio_labels, pred_bio_labels, average='micro')\n    \n    macro_fscore_ent=f1_score(true_ent_labels, pred_ent_labels, average='macro')\n    macro_fscore_bio=f1_score(true_bio_labels, pred_bio_labels, average='macro')\n    \n    return (cm1,cm2, micro_fscore_ent, micro_fscore_bio, macro_fscore_ent, macro_fscore_bio)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.422025Z","iopub.execute_input":"2021-06-18T06:48:17.422548Z","iopub.status.idle":"2021-06-18T06:48:17.441145Z","shell.execute_reply.started":"2021-06-18T06:48:17.422504Z","shell.execute_reply":"2021-06-18T06:48:17.440143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine Tuning","metadata":{}},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model):\n        self.model=model\n        self.iter_count=0\n        self.total_loss=0.0\n        self.ent_loss=0.0\n        self.bios_loss=0.0\n        self.constraint_loss=0.0\n        \n        self.loss_=[]\n        self.ent_loss_=[]\n        self.bios_loss_=[]\n        self.constraint_loss_=[]\n        \n        self.criterion1=nn.BCEWithLogitsLoss(reduction='mean')\n        self.criterion2=nn.CrossEntropyLoss(ignore_index=3, reduction='mean')\n        \n        self.logsoftmax=nn.LogSoftmax(dim=-1)\n        self.logsigmoid=nn.LogSigmoid()\n        self.sigmoid=nn.Sigmoid()\n        \n        self.optimizer=torch.optim.AdamW(model.parameters(), lr=config['max_lr'], \n                                         weight_decay=config['weight_decay'])\n        self.schedular=torch.optim.lr_scheduler.OneCycleLR(self.optimizer,\n                                                           max_lr=config['max_lr'], \n                                                           total_steps=config['num_iterations'])\n        self.labeledIterator=iter(LabeledIterator(config, glove_embeddings, labeled_data))\n    \n    \n    def get_entity_loss(self, slens, y_ents, logyhat_ent0, logyhat_ent1 ):\n        batch_size=y_ents.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            ysum=torch.sum(y_ents[i])\n            seq_len=slens[i].item()\n            if ysum==0:\n                neg_loss+=(-1 * 0.95*logyhat_ent0[i][:seq_len]).sum() + (-1 * 0.05 * logyhat_ent1[i][:seq_len]).sum()\n                neg_cnt+=slens[i]\n                continue\n            for j in range(seq_len):\n                if y_ents[i][j]==0:\n                    neg_loss+=((-1 * 0.95*logyhat_ent0[i][j]) + (-1 * 0.05 * logyhat_ent1[i][j])).sum()\n                    neg_cnt+=1\n                if y_ents[i][j]==1:\n                    pos_loss+=((-1 * 0.95*logyhat_ent1[i][j]) + (-1 * 0.05 * logyhat_ent0[i][j])).sum()\n                    pos_cnt+=1\n                    \n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        loss=pos_loss+neg_loss\n        return loss\n    \n    def get_bios_loss(self, slens, y_bios, logyhat_bios):\n        batch_size=y_bios.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n\n        for i in range(batch_size):\n            seq_len=slens[i].item()\n            ysum=torch.sum(y_bios[i][:seq_len].sum())\n            if ysum==0:\n                neg_loss+=(-1*0.95*logyhat_bios[i][:seq_len][0].sum()) + \\\n                (-1*0.025*logyhat_bios[i][:slens[i]][1].sum()) + \\\n                (-1*0.025*logyhat_bios[i][:slens[i]][2].sum())\n                neg_cnt+=slens[i].item()\n                continue\n                \n            for j in range(slens[i]):\n                if y_bios[i][j]==0:\n                    neg_loss+=(-1*0.95*logyhat_bios[i][j][0].sum()) + \\\n                    (-1*0.025*logyhat_bios[i][j][1].sum()) + \\\n                    (-1*0.025*logyhat_bios[i][j][2].sum())\n                    neg_cnt+=1\n                    \n                elif y_bios[i][j]==1:\n                    pos_loss+=(-1*0.9*logyhat_bios[i][j][1].sum()) + \\\n                    (-1*0.08*logyhat_bios[i][j][2].sum()) + \\\n                    (-1*0.02*logyhat_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    \n                elif y_bios[i][j]==2:\n                    pos_loss+=(-1*0.9*logyhat_bios[i][j][2].sum()) + \\\n                    (-1*0.08*logyhat_bios[i][j][1].sum()) + \\\n                    (-1*0.02*logyhat_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        \n        loss=pos_loss+neg_loss\n        return loss\n    \n    def get_constraint_loss(self, slens, logyhat_bios, logyhat_ent0, logyhat_ent1):\n        loss=torch.tensor(0.0)\n        batch_size=logyhat_bios.shape[0]\n        for i in range(batch_size):\n            cur_loss=torch.tensor(0.0)\n            for j in range(slens[i]):\n                cur_loss+=torch.abs(logyhat_bios[i][j][0] - logyhat_ent0[i][j][0])\n                cur_loss+=torch.abs(logyhat_bios[i][j][1] + logyhat_bios[i][j][2] - logyhat_ent1[i][j][0])\n            loss+=(cur_loss/max(1, slens[i]))\n        loss/=max(batch_size, 1)\n        return loss\n    \n    def train_ops(self, mbs):\n        self.model.train()\n        \n        (X, X_embedd, y_ents, y_bios, slens)=mbs\n        batch_size=X.shape[0]\n        seq_len=X.shape[1]\n        \n        (yhat_ent, yhat_bios)=self.model(X, X_embedd)\n        logyhat_ent1=torch.log(self.sigmoid(yhat_ent))\n        logyhat_ent0=torch.log(1-self.sigmoid(yhat_ent))\n        logyhat_bios=self.logsoftmax(yhat_bios)\n        \n        \n        loss_ents=self.get_entity_loss(slens, y_ents, logyhat_ent0, logyhat_ent1 )\n        loss_bios=self.get_bios_loss(slens, y_bios, logyhat_bios)\n        loss_constraint=self.get_constraint_loss(slens, logyhat_bios, logyhat_ent0, logyhat_ent1)\n        loss = (loss_ents+loss_bios+loss_constraint)/3\n        \n        self.optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(self.model.parameters(), config['grad_clip_val'])\n        self.optimizer.step()\n        self.schedular.step()\n        return (loss.item(), loss_ents.item(), loss_bios.item(), loss_constraint.item())\n    \n    def finetune(self):\n        best_loss=None\n        t1=time.time()\n        self.model.train()\n        while self.iter_count<config['num_iterations']:\n            self.iter_count+=1\n            mbs=next(self.labeledIterator)\n            (loss, loss_ents, loss_bios, loss_constraint) = self.train_ops(mbs)\n            \n            self.total_loss+=loss\n            self.ent_loss+=loss_ents\n            self.bios_loss+=loss_bios\n            self.constraint_loss+=loss_constraint\n\n            \n            self.loss_.append(loss)\n            self.ent_loss_.append(loss_ents)\n            self.bios_loss_.append(loss_bios)\n            self.constraint_loss_.append(loss_constraint)\n        \n            if self.iter_count%config['eval_every']==0:\n                print(\"Evaluating:\")\n                print(\"======\"*10)\n                (cm1,cm2, micro_fscore_ent, micro_fscore_bio, macro_fscore_ent, macro_fscore_bio)=evaluate(model)\n                print(\"Confusion Matrix:\")\n                print(cm1)\n                print(cm2)\n                \n                print(\"Micro F-Score ==> {:.3f} -- {:.3f}\".format(micro_fscore_ent, micro_fscore_bio))\n                print(\"Macro F-Score ==> {:.3f} -- {:.3f}\".format(macro_fscore_ent, macro_fscore_bio))\n                print(\"======\"*10)\n                print()\n                torch.save(model, 'model.pt')\n                \n            if self.iter_count%config['print_every']==0:\n                t2=time.time()\n                print(\"====\"*10)\n                print(\"Iteration:{} | Time Taken:{:.1f}\".format(self.iter_count, (t2-t1)/60))\n                print(\"Total loss:{:.4f}\".format(self.total_loss/config['print_every']))\n                print(\"Entity loss:{:.4f}\".format(self.ent_loss/config['print_every']))\n                print(\"Bios loss:{:.4f}\".format(self.bios_loss/config['print_every']))\n                print(\"Constraint loss:{:.4f}\".format(self.constraint_loss/config['print_every']))\n                t1=time.time()\n                self.total_loss=0\n                self.ent_loss=0\n                self.bios_loss=0\n                self.constraint_loss=0\n                print()\n        print(\"Evaluating:\")\n        print(\"======\"*10)\n        (cm1,cm2, micro_fscore_ent, micro_fscore_bio, macro_fscore_ent, macro_fscore_bio)=evaluate(model)\n        print(\"Confusion Matrix:\")\n        print(cm1)\n        print(cm2)\n\n        print(\"Micro F-Score ==> {:.3f} -- {:.3f}\".format(micro_fscore_ent, micro_fscore_bio))\n        print(\"Macro F-Score ==> {:.3f} -- {:.3f}\".format(macro_fscore_ent, macro_fscore_bio))\n        print(\"======\"*10)\n        print() ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.442752Z","iopub.execute_input":"2021-06-18T06:48:17.443376Z","iopub.status.idle":"2021-06-18T06:48:17.4974Z","shell.execute_reply.started":"2021-06-18T06:48:17.443323Z","shell.execute_reply":"2021-06-18T06:48:17.496231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=torch.load(config['model_path'])\nmodel=model.to(device)\n#trainer=Trainer(model)\n#trainer.finetune()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.499344Z","iopub.execute_input":"2021-06-18T06:48:17.499685Z","iopub.status.idle":"2021-06-18T06:48:17.53434Z","shell.execute_reply.started":"2021-06-18T06:48:17.499652Z","shell.execute_reply":"2021-06-18T06:48:17.533205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"train_df=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df=train_df.groupby('Id')[['dataset_title', 'dataset_label', 'cleaned_label']].agg(list).reset_index()\n\ndef get_all_datalabels(row):\n    dataset_title=row['dataset_title']\n    dataset_label=row['dataset_label']\n    cleaned_label=row['cleaned_label']\n    \n    all_labels=list(set(dataset_label+dataset_title+dataset_title))\n    return all_labels\n    \ntrain_df['all_datalabels']=train_df.apply(get_all_datalabels, axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:17.535888Z","iopub.execute_input":"2021-06-18T06:48:17.536237Z","iopub.status.idle":"2021-06-18T06:48:19.498576Z","shell.execute_reply.started":"2021-06-18T06:48:17.536205Z","shell.execute_reply":"2021-06-18T06:48:19.497439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"database=set()\nfor labels in train_df['all_datalabels'].values:\n    database=database.union(labels)\nprint('Number Of Datasets:', len(database))","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:48:19.500158Z","iopub.execute_input":"2021-06-18T06:48:19.500467Z","iopub.status.idle":"2021-06-18T06:48:19.526919Z","shell.execute_reply.started":"2021-06-18T06:48:19.500435Z","shell.execute_reply":"2021-06-18T06:48:19.525968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Inference:\n    def __init__(self, model):\n        self.model=model\n        self.sigmoid=nn.Sigmoid()\n    def getPredictionLabels(self, dataloader):\n        self.model.eval()\n        predLabels=[]\n        for (slen, X, X_embedd)  in test_dataloader:\n            with torch.no_grad():\n                X=X.to(device)\n                X_embedd=X_embedd.to(device)\n                yent, ybios=self.model(X, X_embedd)\n                yent=yent.detach().cpu()\n                ybios=ybios.detach().cpu()\n                \n                yent=self.sigmoid(yent)\n                batch_size=X.shape[0]\n                ybios=ybios.argmax(dim=-1).numpy()\n                #ybios=ybios.softmax(dim=-1)\n                for i in range(batch_size):\n                    labels=[]\n                    for j in range(slen[i]):\n                        label1=(yent[i][j]>0.6).to(int)\n                        label2=ybios[i][j]\n                        #for k in range(3):\n                            #if ybios[i][j][k]>=0.5:\n                                #label2=k\n                        if label1==1 and (label2==1 or label2==2):\n                            labels.append(label2)\n                        else:\n                            labels.append(0)\n                    predLabels.append(labels)\n        return predLabels","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:01:02.431366Z","iopub.execute_input":"2021-06-18T07:01:02.431733Z","iopub.status.idle":"2021-06-18T07:01:02.442124Z","shell.execute_reply.started":"2021-06-18T07:01:02.4317Z","shell.execute_reply":"2021-06-18T07:01:02.441147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean Dataset","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:01:05.714002Z","iopub.execute_input":"2021-06-18T07:01:05.714491Z","iopub.status.idle":"2021-06-18T07:01:05.720659Z","shell.execute_reply.started":"2021-06-18T07:01:05.714461Z","shell.execute_reply":"2021-06-18T07:01:05.719542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CleanDataset:\n    def __init__(self):\n        pass\n    def clean_text(self, txt):\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower().strip())\n    \n    def is_label_eligible(self, label):\n        label=label.strip()\n        if (label.isnumeric() or label in string.punctuation):\n            return False\n        if len(label)==1:\n            return False\n        if len(label)>1 and label[0].isnumeric():\n            return False\n        if label in ['of', 'and' ,'in', 'or', 'america', 'american', \n                     'england', 'study', 'survey', 'data']:\n            return False\n        return True\n    \n    def predictString(self, row):\n        labels=row['labels']\n        words=row['sentence_words']\n        s=-1;e=-1;\n\n        datalabels=[]\n        for i, label in enumerate(labels):\n            if i==0:\n                continue;\n                \n            if (s==-1) and (labels[i]==1) and (len(words[i])>0) and words[i][0].isupper():\n                s=i;e=i\n            elif labels[i]==2:\n                e=i\n            elif labels[i]==0 and s!=-1:\n                datalabels.append(' '.join(words[s:e+1]) )\n                s=-1; e=-1\n        if s!=-1:\n            datalabels.append(' '.join(words[s:e+1]) )\n        return datalabels\n\n    def process_prediction(self, datalabels, matched_labels):\n        lst=[]\n        for label in datalabels:\n            lst +=label\n        lst+=matched_labels\n        lst=[self.clean_text(l) for l in lst]\n        lst=[l for l in lst if self.is_label_eligible(l)]\n        lst=list(set([l.strip() for l in lst]))\n        \n        filtered_labels=[]\n        for label in sorted(lst, key=len):\n            label = clean_text(label)\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n                filtered_labels.append(label)\n\n        filtered_labels.sort()\n        filtered_labels=filtered_labels[:100]\n        return '|'.join(filtered_labels)\n    \n    def get_datasets(self, test_df, pred_labels, matched_labels):\n        test_df['labels']=pred_labels\n        test_df['PredictionString']=test_df.apply( self.predictString, axis=1)\n        test_df=test_df.groupby('Id')[['PredictionString']].agg(list).reset_index()\n        test_df['PredictionString']=test_df['PredictionString'].apply(self.process_prediction, args=(matched_labels, ))\n        return test_df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:05:45.17774Z","iopub.execute_input":"2021-06-18T07:05:45.17816Z","iopub.status.idle":"2021-06-18T07:05:45.19842Z","shell.execute_reply.started":"2021-06-18T07:05:45.178126Z","shell.execute_reply":"2021-06-18T07:05:45.197215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inferObj=Inference(model)\ncleanDataset=CleanDataset()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:02:32.957009Z","iopub.execute_input":"2021-06-18T07:02:32.957414Z","iopub.status.idle":"2021-06-18T07:02:32.974346Z","shell.execute_reply.started":"2021-06-18T07:02:32.957386Z","shell.execute_reply":"2021-06-18T07:02:32.973469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:02:54.135893Z","iopub.execute_input":"2021-06-18T07:02:54.136251Z","iopub.status.idle":"2021-06-18T07:02:54.141257Z","shell.execute_reply.started":"2021-06-18T07:02:54.136223Z","shell.execute_reply":"2021-06-18T07:02:54.140051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data=[]\nfor idx, pub_id in enumerate(os.listdir(config['test_dir'])):\n    sentences=[]\n    paper=get_publications_data(pub_id, config['test_dir'])\n    \n    #(all_labels, new_entities)=get_datalabels(pub_id)\n    #datalabels=all_labels+new_entities\n    #datalabels=[clean_text(label).strip() for label in datalabels]\n    datalabels=[]\n    for section in paper:\n        sentences+=get_sentences(pub_id, section['text'])\n    if len(sentences)==0:\n        all_data.append({'Id': pub_id.replace('.json', ''), \"PredictionString\": ''})\n        continue\n    \n    data=[]\n    for sentence in sentences:\n        data.append({\n            'Id': pub_id.replace('.json', ''),\n            'sentence_words': sentence\n        })\n    test_df=pd.DataFrame.from_dict(data)\n    test_dataset=TestDataset(test_df,config['max_seq_len'], config['glove_dim'])\n    test_dataloader=torch.utils.data.DataLoader(test_dataset, shuffle=False, \n                                                batch_size=config['test_batch_size'])\n    pred_labels=inferObj.getPredictionLabels(test_dataloader)\n    if len(pred_labels) < len(test_df):\n        for _ in range(len(test_df) - len(pred_labels)):\n            pred_labels.append([])\n    test_df=cleanDataset.get_datasets(test_df, pred_labels, datalabels)\n    all_data.append({'Id': pub_id.replace(\".json\", \"\"), \"PredictionString\": test_df['PredictionString'].values[0]})","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:02:57.262461Z","iopub.execute_input":"2021-06-18T07:02:57.26281Z","iopub.status.idle":"2021-06-18T07:03:00.864657Z","shell.execute_reply.started":"2021-06-18T07:02:57.262781Z","shell.execute_reply":"2021-06-18T07:03:00.863392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_pub_ids=set()\npred_pub_ids=set()\n\nfor pub_id in os.listdir(config['test_dir']):\n    all_pub_ids.add(pub_id.replace('.json', ''))\n\nfor data in all_data:\n    pred_pub_ids.add(data['Id'])\n\nremain_pub_ids=all_pub_ids.difference(pred_pub_ids)\nfor pub_id in remain_pub_ids:\n    all_data.append({'Id': pub_id.replace(\".json\", \"\"), \"PredictionString\":''})\n\n\nsubmission_df=pd.DataFrame.from_dict(all_data)\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:03:05.594011Z","iopub.execute_input":"2021-06-18T07:03:05.594351Z","iopub.status.idle":"2021-06-18T07:03:05.61445Z","shell.execute_reply.started":"2021-06-18T07:03:05.594322Z","shell.execute_reply":"2021-06-18T07:03:05.61305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, p in enumerate(submission_df.PredictionString.values):\n    if idx==4:\n        break\n    print(p)\n    print('=='*10)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:03:08.167809Z","iopub.execute_input":"2021-06-18T07:03:08.168183Z","iopub.status.idle":"2021-06-18T07:03:08.175839Z","shell.execute_reply.started":"2021-06-18T07:03:08.168137Z","shell.execute_reply":"2021-06-18T07:03:08.174688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, pub_id in enumerate(os.listdir(config['test_dir'])):\n    if pub_id!='2f392438-e215-4169-bebf-21ac4ff253e1.json':\n        continue\n    paper=get_publications_data(pub_id, config['test_dir'])\n    for section in paper:\n        text=section['text']\n        for s in sent_tokenize(text):\n            if 'the nces co' in s.lower():\n                print(s)\n                print()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:04:07.500048Z","iopub.execute_input":"2021-06-18T07:04:07.500397Z","iopub.status.idle":"2021-06-18T07:04:07.59528Z","shell.execute_reply.started":"2021-06-18T07:04:07.500368Z","shell.execute_reply":"2021-06-18T07:04:07.594085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}