{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:11:32.563820Z","iopub.execute_input":"2021-06-04T10:11:32.564173Z","iopub.status.idle":"2021-06-04T10:11:32.568010Z","shell.execute_reply.started":"2021-06-04T10:11:32.564142Z","shell.execute_reply":"2021-06-04T10:11:32.567008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:11:32.574445Z","iopub.execute_input":"2021-06-04T10:11:32.574973Z","iopub.status.idle":"2021-06-04T10:12:58.157586Z","shell.execute_reply.started":"2021-06-04T10:11:32.574806Z","shell.execute_reply":"2021-06-04T10:12:58.156708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom scipy.spatial import distance\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T10:12:58.161608Z","iopub.execute_input":"2021-06-04T10:12:58.161876Z","iopub.status.idle":"2021-06-04T10:12:58.167429Z","shell.execute_reply.started":"2021-06-04T10:12:58.161848Z","shell.execute_reply":"2021-06-04T10:12:58.166604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper\n        \nprint(\"Unique Paper Ids: \", str(len(train['Id'].unique())))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:12:58.168912Z","iopub.execute_input":"2021-06-04T10:12:58.169488Z","iopub.status.idle":"2021-06-04T10:13:43.351152Z","shell.execute_reply.started":"2021-06-04T10:12:58.169450Z","shell.execute_reply":"2021-06-04T10:13:43.350247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.352406Z","iopub.execute_input":"2021-06-04T10:13:43.352913Z","iopub.status.idle":"2021-06-04T10:13:43.386680Z","shell.execute_reply.started":"2021-06-04T10:13:43.352873Z","shell.execute_reply":"2021-06-04T10:13:43.386014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.390750Z","iopub.execute_input":"2021-06-04T10:13:43.391012Z","iopub.status.idle":"2021-06-04T10:13:43.409622Z","shell.execute_reply.started":"2021-06-04T10:13:43.390987Z","shell.execute_reply":"2021-06-04T10:13:43.408704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train.index[train['Id']=='c754dec7-c5a3-4337-9892-c02158475064'].tolist()[0]\npapers['c5d5cd2c-59de-4f29-bbb1-6a88c7b52f29']","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.412958Z","iopub.execute_input":"2021-06-04T10:13:43.413296Z","iopub.status.idle":"2021-06-04T10:13:43.423075Z","shell.execute_reply.started":"2021-06-04T10:13:43.413267Z","shell.execute_reply":"2021-06-04T10:13:43.422151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.424466Z","iopub.execute_input":"2021-06-04T10:13:43.425013Z","iopub.status.idle":"2021-06-04T10:13:43.446434Z","shell.execute_reply.started":"2021-06-04T10:13:43.424977Z","shell.execute_reply":"2021-06-04T10:13:43.445475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# finding unique values in each column\n[print(f\"{col}:{len(train[col].unique())}\") for col in train.columns]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.447848Z","iopub.execute_input":"2021-06-04T10:13:43.448294Z","iopub.status.idle":"2021-06-04T10:13:43.472837Z","shell.execute_reply.started":"2021-06-04T10:13:43.448259Z","shell.execute_reply":"2021-06-04T10:13:43.472052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Visualisation","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nfrom nltk.probability import FreqDist\n\nwords = list(train['cleaned_label'].values)\nstopwords=['ourselves', 'hers','the','of','and','in', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:43.474417Z","iopub.execute_input":"2021-06-04T10:13:43.474751Z","iopub.status.idle":"2021-06-04T10:13:44.395770Z","shell.execute_reply.started":"2021-06-04T10:13:43.474718Z","shell.execute_reply":"2021-06-04T10:13:44.394739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1600, height=800, background_color='white', stopwords=STOPWORDS).generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in cleaned_label', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.tight_layout(pad=0)\nplt.title('Freq of 25 Most Common Words in cleaned_label', fontsize=60)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:44.397134Z","iopub.execute_input":"2021-06-04T10:13:44.397480Z","iopub.status.idle":"2021-06-04T10:13:47.086725Z","shell.execute_reply.started":"2021-06-04T10:13:44.397446Z","shell.execute_reply":"2021-06-04T10:13:47.085897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.088094Z","iopub.execute_input":"2021-06-04T10:13:47.088611Z","iopub.status.idle":"2021-06-04T10:13:47.143449Z","shell.execute_reply.started":"2021-06-04T10:13:47.088567Z","shell.execute_reply":"2021-06-04T10:13:47.142574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.144749Z","iopub.execute_input":"2021-06-04T10:13:47.145138Z","iopub.status.idle":"2021-06-04T10:13:47.150358Z","shell.execute_reply.started":"2021-06-04T10:13:47.145101Z","shell.execute_reply":"2021-06-04T10:13:47.149441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.152079Z","iopub.execute_input":"2021-06-04T10:13:47.152455Z","iopub.status.idle":"2021-06-04T10:13:47.264315Z","shell.execute_reply.started":"2021-06-04T10:13:47.152418Z","shell.execute_reply":"2021-06-04T10:13:47.263603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.265796Z","iopub.execute_input":"2021-06-04T10:13:47.266166Z","iopub.status.idle":"2021-06-04T10:13:47.272159Z","shell.execute_reply.started":"2021-06-04T10:13:47.266129Z","shell.execute_reply":"2021-06-04T10:13:47.271230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(literal_preds[:5])\nprint(len(literal_preds))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.273963Z","iopub.execute_input":"2021-06-04T10:13:47.274329Z","iopub.status.idle":"2021-06-04T10:13:47.282596Z","shell.execute_reply.started":"2021-06-04T10:13:47.274293Z","shell.execute_reply":"2021-06-04T10:13:47.281784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\n#SciBERT\nPRETRAINED_PATH = '../input/coleridgescibert/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/coleridgescibert/train_ner.json'\nVAL_PATH = '../input/coleridgescibert/train_ner.json'\n\n#Normal BERT\n# PRETRAINED_PATH = '../input/coleridge-bert-models/output'\n# TEST_INPUT_SAVE_PATH = './input_data'\n# TEST_NER_DATA_FILE = 'test_ner_input.json'\n# TRAIN_PATH = '../input/coleridge-bert-models/train_ner.json'\n# VAL_PATH = '../input/coleridge-bert-models/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.284462Z","iopub.execute_input":"2021-06-04T10:13:47.285015Z","iopub.status.idle":"2021-06-04T10:13:47.291149Z","shell.execute_reply.started":"2021-06-04T10:13:47.284978Z","shell.execute_reply":"2021-06-04T10:13:47.290111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.292695Z","iopub.execute_input":"2021-06-04T10:13:47.293262Z","iopub.status.idle":"2021-06-04T10:13:47.666112Z","shell.execute_reply.started":"2021-06-04T10:13:47.293225Z","shell.execute_reply":"2021-06-04T10:13:47.665191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(train.info)\n#print(train['d0fa7568-7d8e-4db9-870f-f9c6f668c17b'][:5])","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.667338Z","iopub.execute_input":"2021-06-04T10:13:47.667844Z","iopub.status.idle":"2021-06-04T10:13:47.671618Z","shell.execute_reply.started":"2021-06-04T10:13:47.667805Z","shell.execute_reply":"2021-06-04T10:13:47.670745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.672863Z","iopub.execute_input":"2021-06-04T10:13:47.673356Z","iopub.status.idle":"2021-06-04T10:13:47.681623Z","shell.execute_reply.started":"2021-06-04T10:13:47.673321Z","shell.execute_reply":"2021-06-04T10:13:47.680747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.683058Z","iopub.execute_input":"2021-06-04T10:13:47.683436Z","iopub.status.idle":"2021-06-04T10:13:47.730860Z","shell.execute_reply.started":"2021-06-04T10:13:47.683364Z","shell.execute_reply":"2021-06-04T10:13:47.730138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(test_rows[35])\nprint(paper_length)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.731946Z","iopub.execute_input":"2021-06-04T10:13:47.732272Z","iopub.status.idle":"2021-06-04T10:13:47.737603Z","shell.execute_reply.started":"2021-06-04T10:13:47.732238Z","shell.execute_reply":"2021-06-04T10:13:47.736494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.738867Z","iopub.execute_input":"2021-06-04T10:13:47.739491Z","iopub.status.idle":"2021-06-04T10:13:47.747395Z","shell.execute_reply.started":"2021-06-04T10:13:47.739426Z","shell.execute_reply":"2021-06-04T10:13:47.746439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:47.748621Z","iopub.execute_input":"2021-06-04T10:13:47.748974Z","iopub.status.idle":"2021-06-04T10:13:48.543691Z","shell.execute_reply.started":"2021-06-04T10:13:47.748926Z","shell.execute_reply":"2021-06-04T10:13:48.542550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:48.545547Z","iopub.execute_input":"2021-06-04T10:13:48.545903Z","iopub.status.idle":"2021-06-04T10:13:48.552924Z","shell.execute_reply.started":"2021-06-04T10:13:48.545864Z","shell.execute_reply":"2021-06-04T10:13:48.550990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:13:48.554438Z","iopub.execute_input":"2021-06-04T10:13:48.554924Z","iopub.status.idle":"2021-06-04T10:15:05.653681Z","shell.execute_reply.started":"2021-06-04T10:13:48.554887Z","shell.execute_reply":"2021-06-04T10:15:05.652895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n#print(test_sentences)\n#del test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:05.656194Z","iopub.execute_input":"2021-06-04T10:15:05.656462Z","iopub.status.idle":"2021-06-04T10:15:05.664723Z","shell.execute_reply.started":"2021-06-04T10:15:05.656435Z","shell.execute_reply":"2021-06-04T10:15:05.664011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n#print(len(test_sentences))\n#print(\"paper length: \", str(paper_length))\nfor length in paper_length:\n    #print(\"\\n \\n Starting new paper\")\n    labels = set()\n    #print(\"bert outputs \", str(bert_outputs[:length]))\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                #print(\"Sentence is: \", str(sentence))\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n        \n    \n    # record dataset labels for this publication\n    #print(\"adding \", str(labels), \" to bert labels\")\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:05.667596Z","iopub.execute_input":"2021-06-04T10:15:05.667842Z","iopub.status.idle":"2021-06-04T10:15:06.261127Z","shell.execute_reply.started":"2021-06-04T10:15:05.667818Z","shell.execute_reply":"2021-06-04T10:15:06.260148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bert_dataset_labels[:5]\n#print(len(test_sentences))\n#print(bert_outputs)\n#print(this_preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.262770Z","iopub.execute_input":"2021-06-04T10:15:06.263114Z","iopub.status.idle":"2021-06-04T10:15:06.271156Z","shell.execute_reply.started":"2021-06-04T10:15:06.263086Z","shell.execute_reply":"2021-06-04T10:15:06.270244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    sim = float(intersection) / union\n    #printf(\"string1: %s string2: %s, jac_sim: %d\", s1, s2, sim)\n    return sim\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n        #if len(filtered) == 0 or all(distance.sokalmichener(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.273544Z","iopub.execute_input":"2021-06-04T10:15:06.273884Z","iopub.status.idle":"2021-06-04T10:15:06.283513Z","shell.execute_reply.started":"2021-06-04T10:15:06.273851Z","shell.execute_reply":"2021-06-04T10:15:06.282527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(filtered_bert_labels[:5])\n# print(literal_preds[:5])\nfiltered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.285088Z","iopub.execute_input":"2021-06-04T10:15:06.285504Z","iopub.status.idle":"2021-06-04T10:15:06.295687Z","shell.execute_reply.started":"2021-06-04T10:15:06.285467Z","shell.execute_reply":"2021-06-04T10:15:06.294747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"# final_predictions = []\n# for literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n#     if literal_match:\n#         final_predictions.append(literal_match)\n#     else:\n#         print(\"we used BERT\")\n#         final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.297437Z","iopub.execute_input":"2021-06-04T10:15:06.297804Z","iopub.status.idle":"2021-06-04T10:15:06.303727Z","shell.execute_reply.started":"2021-06-04T10:15:06.297768Z","shell.execute_reply":"2021-06-04T10:15:06.303075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\nfor literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n    #print(\"\\n literal match: \", str(literal_match))\n   # print(\"bert pred: \", str(bert_pred))\n    if bert_pred:\n     #   print(\"we used BERT\")\n        final_predictions.append(bert_pred)\n    else:\n        print(\"we used Literal Matching\")\n        final_predictions.append(literal_match)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.308884Z","iopub.execute_input":"2021-06-04T10:15:06.309164Z","iopub.status.idle":"2021-06-04T10:15:06.315159Z","shell.execute_reply.started":"2021-06-04T10:15:06.309130Z","shell.execute_reply":"2021-06-04T10:15:06.313955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"scibert predictions:\n['alzheimer s disease neuroimaging initiative adni',\n 'common core of data|alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study',\n 'common core of data|alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study',\n 'common core of data|alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study']\n\nbert predictions:\n['alzheimer s disease neuroimaging initiative adni',\n 'alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study',\n 'alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study',\n 'alzheimer s disease neuroimaging initiative adni|trends in international mathematics and science study']","metadata":{}},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.316854Z","iopub.execute_input":"2021-06-04T10:15:06.317485Z","iopub.status.idle":"2021-06-04T10:15:06.330080Z","shell.execute_reply.started":"2021-06-04T10:15:06.317449Z","shell.execute_reply":"2021-06-04T10:15:06.329231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.331784Z","iopub.execute_input":"2021-06-04T10:15:06.332531Z","iopub.status.idle":"2021-06-04T10:15:06.450101Z","shell.execute_reply.started":"2021-06-04T10:15:06.332463Z","shell.execute_reply":"2021-06-04T10:15:06.449379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(final_predictions)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T10:15:06.451692Z","iopub.execute_input":"2021-06-04T10:15:06.452197Z","iopub.status.idle":"2021-06-04T10:15:06.455993Z","shell.execute_reply.started":"2021-06-04T10:15:06.452161Z","shell.execute_reply":"2021-06-04T10:15:06.455013Z"},"trusted":true},"execution_count":null,"outputs":[]}]}