{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:07:48.100795Z","iopub.execute_input":"2021-06-07T11:07:48.101177Z","iopub.status.idle":"2021-06-07T11:07:48.105469Z","shell.execute_reply.started":"2021-06-07T11:07:48.101145Z","shell.execute_reply":"2021-06-07T11:07:48.104333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:07:48.111391Z","iopub.execute_input":"2021-06-07T11:07:48.111801Z","iopub.status.idle":"2021-06-07T11:09:15.332746Z","shell.execute_reply.started":"2021-06-07T11:07:48.111771Z","shell.execute_reply":"2021-06-07T11:09:15.331615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-07T11:09:15.334898Z","iopub.execute_input":"2021-06-07T11:09:15.335227Z","iopub.status.idle":"2021-06-07T11:09:15.344655Z","shell.execute_reply.started":"2021-06-07T11:09:15.335198Z","shell.execute_reply":"2021-06-07T11:09:15.343816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"COMPUTE_EVAL = True","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:15.34646Z","iopub.execute_input":"2021-06-07T11:09:15.346885Z","iopub.status.idle":"2021-06-07T11:09:15.352493Z","shell.execute_reply.started":"2021-06-07T11:09:15.346847Z","shell.execute_reply":"2021-06-07T11:09:15.351246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_EVAL:\n    df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n    df['split'] = np.random.randn(df.shape[0], 1)\n\n    msk = np.random.rand(len(df)) <= 0.9\n\n    train = df[msk]\n    train2 = train\n    val = df[~msk]\nelse:\n    train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\n    train = pd.read_csv(train_path)\n    train = train[:MAX_SAMPLE]\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:15.354458Z","iopub.execute_input":"2021-06-07T11:09:15.354981Z","iopub.status.idle":"2021-06-07T11:09:23.123349Z","shell.execute_reply.started":"2021-06-07T11:09:15.354944Z","shell.execute_reply":"2021-06-07T11:09:23.122536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_EVAL:\n    sample_submission = val\n    paper_test_folder = paper_train_folder\nelse: \n    sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\n    sample_submission = pd.read_csv(sample_submission_path)\n\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:23.12661Z","iopub.execute_input":"2021-06-07T11:09:23.126951Z","iopub.status.idle":"2021-06-07T11:09:24.217375Z","shell.execute_reply.started":"2021-06-07T11:09:23.126916Z","shell.execute_reply":"2021-06-07T11:09:24.216455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:24.219859Z","iopub.execute_input":"2021-06-07T11:09:24.220226Z","iopub.status.idle":"2021-06-07T11:09:24.270142Z","shell.execute_reply.started":"2021-06-07T11:09:24.220188Z","shell.execute_reply":"2021-06-07T11:09:24.269242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:24.271397Z","iopub.execute_input":"2021-06-07T11:09:24.271902Z","iopub.status.idle":"2021-06-07T11:09:24.277501Z","shell.execute_reply.started":"2021-06-07T11:09:24.271864Z","shell.execute_reply":"2021-06-07T11:09:24.2766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:24.278648Z","iopub.execute_input":"2021-06-07T11:09:24.27921Z","iopub.status.idle":"2021-06-07T11:09:55.021431Z","shell.execute_reply.started":"2021-06-07T11:09:24.279058Z","shell.execute_reply":"2021-06-07T11:09:55.020663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:55.022866Z","iopub.execute_input":"2021-06-07T11:09:55.023226Z","iopub.status.idle":"2021-06-07T11:09:55.029026Z","shell.execute_reply.started":"2021-06-07T11:09:55.023189Z","shell.execute_reply":"2021-06-07T11:09:55.028171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/bert-standard-90-training-data/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/bert-standard-90-training-data/train_ner.json'\nVAL_PATH = '../input/bert-standard-90-training-data/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:55.0309Z","iopub.execute_input":"2021-06-07T11:09:55.031468Z","iopub.status.idle":"2021-06-07T11:09:55.041777Z","shell.execute_reply.started":"2021-06-07T11:09:55.031424Z","shell.execute_reply":"2021-06-07T11:09:55.040843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:55.043193Z","iopub.execute_input":"2021-06-07T11:09:55.043742Z","iopub.status.idle":"2021-06-07T11:09:55.379868Z","shell.execute_reply.started":"2021-06-07T11:09:55.043702Z","shell.execute_reply":"2021-06-07T11:09:55.378962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:55.381172Z","iopub.execute_input":"2021-06-07T11:09:55.381685Z","iopub.status.idle":"2021-06-07T11:09:55.388787Z","shell.execute_reply.started":"2021-06-07T11:09:55.381647Z","shell.execute_reply":"2021-06-07T11:09:55.388053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:09:55.390033Z","iopub.execute_input":"2021-06-07T11:09:55.390675Z","iopub.status.idle":"2021-06-07T11:10:09.33177Z","shell.execute_reply.started":"2021-06-07T11:09:55.390633Z","shell.execute_reply":"2021-06-07T11:10:09.330851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:10:09.333085Z","iopub.execute_input":"2021-06-07T11:10:09.333588Z","iopub.status.idle":"2021-06-07T11:10:09.33917Z","shell.execute_reply.started":"2021-06-07T11:10:09.333549Z","shell.execute_reply":"2021-06-07T11:10:09.338473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:10:09.342357Z","iopub.execute_input":"2021-06-07T11:10:09.342625Z","iopub.status.idle":"2021-06-07T11:10:10.012601Z","shell.execute_reply.started":"2021-06-07T11:10:09.3426Z","shell.execute_reply":"2021-06-07T11:10:10.011684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:10:10.014095Z","iopub.execute_input":"2021-06-07T11:10:10.014431Z","iopub.status.idle":"2021-06-07T11:10:10.020862Z","shell.execute_reply.started":"2021-06-07T11:10:10.014396Z","shell.execute_reply":"2021-06-07T11:10:10.020098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:10:10.022285Z","iopub.execute_input":"2021-06-07T11:10:10.02283Z","iopub.status.idle":"2021-06-07T11:21:45.901743Z","shell.execute_reply.started":"2021-06-07T11:10:10.02279Z","shell.execute_reply":"2021-06-07T11:21:45.900521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:45.90673Z","iopub.execute_input":"2021-06-07T11:21:45.908948Z","iopub.status.idle":"2021-06-07T11:21:45.986592Z","shell.execute_reply.started":"2021-06-07T11:21:45.908879Z","shell.execute_reply":"2021-06-07T11:21:45.985603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:45.990474Z","iopub.execute_input":"2021-06-07T11:21:45.990733Z","iopub.status.idle":"2021-06-07T11:21:46.695245Z","shell.execute_reply.started":"2021-06-07T11:21:45.990707Z","shell.execute_reply":"2021-06-07T11:21:46.69448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.696593Z","iopub.execute_input":"2021-06-07T11:21:46.69698Z","iopub.status.idle":"2021-06-07T11:21:46.703852Z","shell.execute_reply.started":"2021-06-07T11:21:46.696936Z","shell.execute_reply":"2021-06-07T11:21:46.703025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.705234Z","iopub.execute_input":"2021-06-07T11:21:46.705729Z","iopub.status.idle":"2021-06-07T11:21:46.729311Z","shell.execute_reply.started":"2021-06-07T11:21:46.705691Z","shell.execute_reply":"2021-06-07T11:21:46.728637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.730429Z","iopub.execute_input":"2021-06-07T11:21:46.730794Z","iopub.status.idle":"2021-06-07T11:21:46.74591Z","shell.execute_reply.started":"2021-06-07T11:21:46.730756Z","shell.execute_reply":"2021-06-07T11:21:46.74503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"counter = 0\nfinal_predictions = []\nfor literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n    if bert_pred:\n        final_predictions.append(bert_pred)\n        \n    else:\n        #print(\"we used Literal Matching\")\n        final_predictions.append(literal_match)\n        counter = counter+1","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.747676Z","iopub.execute_input":"2021-06-07T11:21:46.748059Z","iopub.status.idle":"2021-06-07T11:21:46.755874Z","shell.execute_reply.started":"2021-06-07T11:21:46.748021Z","shell.execute_reply":"2021-06-07T11:21:46.755166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.757515Z","iopub.execute_input":"2021-06-07T11:21:46.757849Z","iopub.status.idle":"2021-06-07T11:21:46.783444Z","shell.execute_reply.started":"2021-06-07T11:21:46.757812Z","shell.execute_reply":"2021-06-07T11:21:46.782307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = {'cleaned_label': sample_submission['cleaned_label'], 'Prediction_string': final_predictions}\ndict(list(t.items())[:0])\n#print(final_predictions[350])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:37:52.17548Z","iopub.execute_input":"2021-06-07T11:37:52.175799Z","iopub.status.idle":"2021-06-07T11:37:52.180719Z","shell.execute_reply.started":"2021-06-07T11:37:52.175768Z","shell.execute_reply":"2021-06-07T11:37:52.179838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getMetric(col):\n    def f1score(row):\n        n = len(np.intersect1d(row.PredictionString.split('|'), row[col]))\n        return 2*n / (len(row.PredictionString.split('|')) + len(row[col]))\n    return f1score\n\ndef my_jaccard(strs): \n    str1, str2 = strs\n    temp_list = []\n    for sentence in str1.lower().split('|'):\n        a = set(str1.lower().split()) \n        b = set(str2.lower().split())\n        c = a.intersection(b)\n        d = float(len(c)) / (len(a) + len(b) - len(c))\n        temp_list.append(d)\n    return sum(temp_list) / len(temp_list)   ","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:22:32.576698Z","iopub.execute_input":"2021-06-07T11:22:32.577018Z","iopub.status.idle":"2021-06-07T11:22:32.586222Z","shell.execute_reply.started":"2021-06-07T11:22:32.576987Z","shell.execute_reply":"2021-06-07T11:22:32.584969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_EVAL:\n    getMetric_score = sample_submission.apply(getMetric('cleaned_label'), axis=1)\n    print('getMetric_score =', getMetric_score.mean())\n    my_jaccard_score = sample_submission[['PredictionString', 'cleaned_label']].apply(my_jaccard, axis=1)\n    print('my_jaccard_score =', my_jaccard_score.mean())\n    print('amount of times literal matching is used: ', counter)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:22:44.080785Z","iopub.execute_input":"2021-06-07T11:22:44.081122Z","iopub.status.idle":"2021-06-07T11:22:44.244426Z","shell.execute_reply.started":"2021-06-07T11:22:44.08109Z","shell.execute_reply":"2021-06-07T11:22:44.243505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:21:46.784959Z","iopub.execute_input":"2021-06-07T11:21:46.785429Z","iopub.status.idle":"2021-06-07T11:21:46.934244Z","shell.execute_reply.started":"2021-06-07T11:21:46.785383Z","shell.execute_reply":"2021-06-07T11:21:46.933479Z"},"trusted":true},"execution_count":null,"outputs":[]}]}