{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\n\nfrom spacy.lang.en import English\nfrom spacy.lang.en.stop_words import STOP_WORDS\nfrom nltk.tokenize import word_tokenize, sent_tokenize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T13:20:34.58955Z","iopub.execute_input":"2021-06-04T13:20:34.590676Z","iopub.status.idle":"2021-06-04T13:20:35.814451Z","shell.execute_reply.started":"2021-06-04T13:20:34.590485Z","shell.execute_reply":"2021-06-04T13:20:35.812934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df=train_df.groupby('Id')[['dataset_title', 'dataset_label']].agg(list).reset_index()\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:35.817119Z","iopub.execute_input":"2021-06-04T13:20:35.817633Z","iopub.status.idle":"2021-06-04T13:20:37.158149Z","shell.execute_reply.started":"2021-06-04T13:20:35.81759Z","shell.execute_reply":"2021-06-04T13:20:37.15686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_datalabels(row):\n    dataset_title=row['dataset_title']\n    dataset_label=row['dataset_label']\n    \n    all_labels=list(set(dataset_label+dataset_title))\n    return all_labels\n    \ntrain_df['all_datalabels']=train_df.apply(get_all_datalabels, axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.160201Z","iopub.execute_input":"2021-06-04T13:20:37.160707Z","iopub.status.idle":"2021-06-04T13:20:37.475703Z","shell.execute_reply.started":"2021-06-04T13:20:37.160659Z","shell.execute_reply":"2021-06-04T13:20:37.473762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"database=set()\nfor labels in train_df['all_datalabels'].values:\n    database=database.union(labels)\nprint('Number Of Datasets:', len(database))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.477586Z","iopub.execute_input":"2021-06-04T13:20:37.478189Z","iopub.status.idle":"2021-06-04T13:20:37.506957Z","shell.execute_reply.started":"2021-06-04T13:20:37.478128Z","shell.execute_reply":"2021-06-04T13:20:37.505225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_data(pub_id):\n    pub_filename='../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(pub_id)\n    with open(pub_filename) as file:\n        data=json.load(file)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.51135Z","iopub.execute_input":"2021-06-04T13:20:37.512065Z","iopub.status.idle":"2021-06-04T13:20:37.519475Z","shell.execute_reply.started":"2021-06-04T13:20:37.511975Z","shell.execute_reply":"2021-06-04T13:20:37.517897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_word_properties(word):\n    prop={}\n    \n    prop['word']=word\n    prop['is_alpha']=False\n    prop['is_title']=False\n    prop['is_upper']=False\n    prop['is_lower']=False\n    prop['has_upper']=False\n    prop['is_number']=False\n    prop['is_stopword']=False\n    prop['is_punct'] = False\n    prop['alpha_1']=False\n    \n    \n    if word.isalpha():\n        prop['is_alpha']=True\n    if word[0].isalpha():\n        prop['alpha_1']=True\n    if word.islower():\n        prop['is_lower']=True\n    if word.isupper():\n        prop['is_upper']=True\n        prop['has_upper']=True\n    if word.lower() in STOP_WORDS:\n        prop['is_stopword']=True\n    if word.istitle():\n        prop['is_title']=True\n        prop['has_upper']=True\n    if word.isnumeric():\n        prop['is_number']=True\n        \n    if word in string.punctuation:\n        prop['is_punct']=True\n    \n    if (prop['is_alpha']) and (not prop['has_upper']) and (not prop['is_lower']):\n        for ch in word:\n            if ch.isupper():\n                prop['has_upper']=True\n    return prop\n\n\ndef get_candidate_index(i, word_props, word_len, candidates):\n    word=word_props[i]['word'].lower()\n    if (candidates[i-1] == 0) and  (word_props[i]['has_upper']) and (not word_props[i]['is_stopword']):\n        return 1\n    \n    if candidates[i-1]==1 or candidates[i-1]==2:\n        if word_props[i]['is_punct'] and word in ['(', ')', '-']:\n            return 2\n        if word_props[i]['is_punct']:\n            return 0\n        if word_props[i]['has_upper']:\n            return 2\n        if word_props[i]['is_lower'] and (i+1 < word_len) and (word_props[i+1]['is_lower']):\n            return 0\n        if (word_props[i]['is_lower'] and (i+1 < word_len) and \n                (word_props[i+1]['has_upper']) and word in ['in', 'for', 'of', 'the', 'and']):\n            return 2\n    return 0\n    \ndef get_candidate_entities(sentence):\n    words=word_tokenize(sentence)\n    words=[word.strip() for word in words]\n    words=[word for word in words if len(word)>0]\n    if words[0].isnumeric():\n        words=words[1:]\n    words_len=len(words)\n    word_props=[]\n    candidates=[0]*words_len\n    cwords=[]\n    if words_len <= 5:\n        return (candidates, cwords)\n    \n    for word in words:\n        prop=get_word_properties(word)\n        word_props.append(prop)\n    \n    if (not word_props[0]['is_stopword']) and (word_props[0]['has_upper'] and (not word_props[1]['word']==',')):\n        candidates[0]=1\n    \n    for i in range(1, words_len):\n        prop=word_props[i]\n        word=prop['word']\n        candidates[i]=get_candidate_index(i, word_props, words_len, candidates)\n    \n    #Removing the first word sequence as candidate words.\n    if candidates[0]==1:\n        candidates[0]=0\n        for i in range(1, words_len):\n            if candidates[i]==1:\n                break\n            candidates[i]=0\n    s=-1; e=-1\n    for i in range(words_len):\n        if candidates[i]==1 and s==-1:\n            s=i\n            e=i\n        elif candidates[i]==1 and s!=-1:\n            cwords.append(' '.join(words[s: e+1]))\n            s=i\n        elif candidates[i]==2:\n            e=i\n        elif s!=-1 and candidates[i]==0:\n            cwords.append(' '.join(words[s: e+1]))\n            s=-1\n            e=-1\n    cwords=[word for word in cwords if (len(word)>2) and (not word[0].isnumeric())]\n    return (candidates, cwords)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.521789Z","iopub.execute_input":"2021-06-04T13:20:37.522296Z","iopub.status.idle":"2021-06-04T13:20:37.553377Z","shell.execute_reply.started":"2021-06-04T13:20:37.522251Z","shell.execute_reply":"2021-06-04T13:20:37.551819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_publications_data(pub_id):\n    pub_filename='../input/coleridgeinitiative-show-us-the-data/test/{}'.format(pub_id)\n    with open(pub_filename) as file:\n        data=json.load(file)\n    return data\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.555215Z","iopub.execute_input":"2021-06-04T13:20:37.555673Z","iopub.status.idle":"2021-06-04T13:20:37.577558Z","shell.execute_reply.started":"2021-06-04T13:20:37.555627Z","shell.execute_reply":"2021-06-04T13:20:37.575898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_abbrevation(sentence_words, sentence_len, i):\n    if i==sentence_len or sentence_words[i]!='(':\n        return ''\n    else:\n        j=i\n        while j<sentence_len:\n            if sentence_words[j] ==')':\n                break\n            j+=1\n        abbr=sentence_words[i+1: j]\n        abbr=[word.strip() for word in abbr]\n        abbr=[word for word in abbr if len(word)!=0]\n        if len(abbr) ==1 and abbr[0] == 1:\n            return ''\n        if not abbr[0][0].isupper():\n            return ''\n    return ''.join(sentence_words[i: j+1])\n        \n\ndef get_abbrevated_labels(sentence, all_labels):\n    sentence_words=word_tokenize(sentence)\n    sentence_words=[w for w in sentence_words if w!='']\n    sentence_len=len(sentence_words)\n    abbr_labels=[]\n    for label in all_labels:\n        label_words=word_tokenize(label)\n        for i, sword in enumerate(sentence_words):\n            flag=True\n            for j, lword in enumerate(label_words):\n                if sentence_words[i+j] != lword:\n                    flag=False\n                    break\n            if flag:\n                abbr=get_abbrevation(sentence_words, sentence_len, i+len(label_words))\n                abbr_labels.append(label+\" \"+abbr)\n                break\n    return abbr_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:20:37.580748Z","iopub.execute_input":"2021-06-04T13:20:37.581293Z","iopub.status.idle":"2021-06-04T13:20:37.598496Z","shell.execute_reply.started":"2021-06-04T13:20:37.581249Z","shell.execute_reply":"2021-06-04T13:20:37.597076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exclude_entities=['table', 'fig', 'provide','data','mri', 'result']","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:42:49.504787Z","iopub.execute_input":"2021-06-04T13:42:49.505169Z","iopub.status.idle":"2021-06-04T13:42:49.511328Z","shell.execute_reply.started":"2021-06-04T13:42:49.505137Z","shell.execute_reply":"2021-06-04T13:42:49.509951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_new_entities(candidates, all_labels, sentence):\n    new_ents=[]\n    if len(all_labels)==0:\n        return []\n    sent_words=word_tokenize(sentence)\n    words_len=len(sent_words)\n    entity_markers=[0]*words_len\n    ent_id=1\n    \n    try:\n        for label in all_labels:\n            lwords=word_tokenize(label)\n            for i in range(words_len):\n                flag=True\n                for j in range(len(lwords)):\n                    if lwords[j] != sent_words[i+j]:\n                        flag=False\n                        break\n                if flag:\n                    for k in range(i, i+len(lwords)):\n                        entity_markers[k]=ent_id\n                    ent_id+=1\n        for i, em in enumerate(entity_markers):\n            if (em==0):\n                continue\n            tol=1\n            if (i-1)>=0 and entity_markers[i-1]==0 and sent_words[i-1]==',':\n                s=-1;e=-1\n                for j in range(i-2, -1, -1):\n                    if tol==0 or entity_markers[j]!=0:\n                        s=-1;e=-1;\n                        break\n                    if e==-1 and candidates[j]==0 and sent_words[j]!=')':\n                        tol-=1\n\n                    if s!=-1 and candidates[j]==1:\n                        s=j\n                        break\n\n                    if e==-1 and candidates[j]!=0:\n                        e=j;s=j;tol=2;\n                        if candidates[j]==1:\n                            break\n\n                    if candidates[j]==2:\n                        s=j\n                if s!=-1:\n                    new_ents.append(sent_words[s:e+1])\n\n\n            tol=1\n            if (i+1)<words_len and entity_markers[i+1]==0 and (sent_words[i+1]==',' or sent_words[i+1]=='and'):\n                s=-1; e=-1\n                for j in range(i+2, words_len):\n                    if tol==0 or entity_markers[j]!=0:\n                        s=-1; e=-1\n                        break\n                    if s==-1 and candidates[j]==0 and sent_words[j]!='(':\n                        tol-=1\n                    if s==-1 and candidates[j]!=0:\n                        s=j; e=j; tol=2\n                    if s!=-1 and (candidates[j]==1 or candidates[j]==0):\n                        e=j-1\n                        break\n                    else:\n                        e=j\n                if s!=-1:\n                    new_ents.append(sent_words[s:e+1])\n    except:\n        pass\n    \n    new_ents_final=[]\n    for ent in new_ents:\n        flag=True\n        for f in exclude_entities:\n            if f in ent:\n                flag=False\n                break\n        if flag:\n            new_ents_final.append(ent)\n    return new_ents_final\n\ndef get_datalabels(pub_id):\n    data=get_publications_data(pub_id)\n    all_labels=[]\n    new_entities=[]\n    for section in data:\n        text=section['text']\n        sentences=sent_tokenize(text)\n        for sentence in sentences:\n            sentence_labels=[]\n            for label in database:\n                if label in sentence:\n                    sentence_labels.append(label)\n            if len(sentence_labels) != 0:\n                abrevation_labels=get_abbrevated_labels(sentence, sentence_labels)\n                all_labels+=sentence_labels\n                all_labels+=abrevation_labels\n                \n                #Taken from https://www.kaggle.com/tungmphung/pytorch-bert-for-named-entity-recognition/comments\n                if any( [word in sentence.lower() for word in ['data', 'study', 'database','taken from']] ): \n                    (candidates, cwords)=get_candidate_entities(sentence)\n                    new_entities+=get_new_entities(candidates, all_labels, sentence)\n    return (all_labels, new_entities)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:43:36.365465Z","iopub.execute_input":"2021-06-04T13:43:36.365918Z","iopub.status.idle":"2021-06-04T13:43:36.390936Z","shell.execute_reply.started":"2021-06-04T13:43:36.36588Z","shell.execute_reply":"2021-06-04T13:43:36.389709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"all_data=[]\nfor filename in os.listdir('../input/coleridgeinitiative-show-us-the-data/test'):\n    (all_labels, new_entities)=get_datalabels(filename)\n    datalabels=all_labels+new_entities\n    Id=filename.replace('.json', '')\n    datalabels=list(set([clean_text(label).strip() for label in datalabels]))\n    datalabels.sort()\n    \n    predictionString='|'.join(datalabels)\n    all_data.append({\n        'Id': Id,\n        'PredictionString': predictionString\n    })","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:32:40.515077Z","iopub.execute_input":"2021-06-04T13:32:40.515729Z","iopub.status.idle":"2021-06-04T13:32:40.764625Z","shell.execute_reply.started":"2021-06-04T13:32:40.515672Z","shell.execute_reply":"2021-06-04T13:32:40.763715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df=pd.DataFrame.from_dict(all_data)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:32:11.744359Z","iopub.execute_input":"2021-06-04T13:32:11.744938Z","iopub.status.idle":"2021-06-04T13:32:11.758569Z","shell.execute_reply.started":"2021-06-04T13:32:11.744901Z","shell.execute_reply":"2021-06-04T13:32:11.757486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:32:14.393596Z","iopub.execute_input":"2021-06-04T13:32:14.394225Z","iopub.status.idle":"2021-06-04T13:32:14.400272Z","shell.execute_reply.started":"2021-06-04T13:32:14.394166Z","shell.execute_reply":"2021-06-04T13:32:14.399202Z"},"trusted":true},"execution_count":null,"outputs":[]}]}