{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\nTrained a **New MLM** and added some **New Labels** based on [External_Datasets_Matching + MLMv4](https://www.kaggle.com/chienhsianghung/external-datasets-matching-mlmv4).<br>\nLet's build a Non-Train-Matching, Train-Label-Removed LB together. Come, Enjoy, and Share! -> [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/244505)\n\n**New MLM**\n- [[Coleridge] BERT - MLMv4](https://www.kaggle.com/chienhsianghung/coleridge-bert-mlmv4)\n\n**New Labels**\n- [Coleridge additional_gov_datasets_22000popular](https://www.kaggle.com/chienhsianghung/coleridge-additional-gov-datasets-22000popular)<br>\nWhere and How I scrapped external datasets: [web scraping for bigger_govt_dataset_list](https://www.kaggle.com/chienhsianghung/web-scraping-for-bigger-govt-dataset-list)\n\nAgain, special thanks to [@tungmphung](https://www.kaggle.com/tungmphung) and [@mlconsult](https://www.kaggle.com/mlconsult) who are generous to share their knowledge.\n\nSome previous works and resources:\n- [Coleridge Initiative| Naive CV (Local Validation)](https://www.kaggle.com/chienhsianghung/coleridge-initiative-naive-cv-local-validation)\n- [Matching + Baseline + MLM w/ Evaluation Metric](https://www.kaggle.com/chienhsianghung/matching-baseline-mlm-w-evaluation-metric)\n- [External_Datasets_Matching + MLM](https://www.kaggle.com/chienhsianghung/external-datasets-matching-mlm)","metadata":{}},{"cell_type":"markdown","source":"# Setting","metadata":{"papermill":{"duration":0.017221,"end_time":"2021-05-22T12:27:26.694629","exception":false,"start_time":"2021-05-22T12:27:26.677408","status":"completed"},"tags":[]}},{"cell_type":"code","source":"COMPUTE_CV = False\nMATCH_ONLY = False\nMLM_ONLY = False\nSEED = 42\n\nADNL_GOVT_LABELS_PATH = '../input/coleridge-additional-datasets/additional_datasets_v6.csv'\n\nif not MATCH_ONLY:\n    PRETRAINED_PATH = '../input/coleridge-bert-mlmv4/output-mlm/checkpoint-48000'\n    TOKENIZER_PATH = '../input/coleridge-bert-mlmv4/model_tokenizer'\n\n    MAX_LENGTH = 64\n    OVERLAP = 20\n\n    PREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\n    DATASET_SYMBOL = '$' # this symbol represents a dataset name\n    NONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"papermill":{"duration":0.024946,"end_time":"2021-05-22T12:27:26.735674","exception":false,"start_time":"2021-05-22T12:27:26.710728","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:10:32.090823Z","iopub.execute_input":"2021-06-02T07:10:32.091672Z","iopub.status.idle":"2021-06-02T07:10:32.098015Z","shell.execute_reply.started":"2021-06-02T07:10:32.091562Z","shell.execute_reply":"2021-06-02T07:10:32.096772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\n\nimport os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.autonotebook import tqdm\nfrom datasets import load_dataset\nimport torch\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\n\nfrom IPython.display import clear_output\n\n\nclear_output()","metadata":{"_kg_hide-output":true,"papermill":{"duration":94.757349,"end_time":"2021-05-22T12:29:01.508696","exception":false,"start_time":"2021-05-22T12:27:26.751347","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:10:32.100152Z","iopub.execute_input":"2021-06-02T07:10:32.100709Z","iopub.status.idle":"2021-06-02T07:12:17.957275Z","shell.execute_reply.started":"2021-06-02T07:10:32.100657Z","shell.execute_reply":"2021-06-02T07:12:17.956139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://huggingface.co/transformers/_modules/transformers/trainer_utils.html\ndef set_seed(seed: int):\n    \"\"\"\n    Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch`` and/or ``tf`` (if\n    installed).\n\n    Args:\n        seed (:obj:`int`): The seed to set.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # ^^ safe to call this function even if cuda is not available\n    \n    print(f'Setted Pipeline SEED = {SEED}')\n\n\nset_seed(SEED)","metadata":{"papermill":{"duration":0.029127,"end_time":"2021-05-22T12:29:01.553927","exception":false,"start_time":"2021-05-22T12:29:01.5248","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:17.959115Z","iopub.execute_input":"2021-06-02T07:12:17.959467Z","iopub.status.idle":"2021-06-02T07:12:17.973844Z","shell.execute_reply.started":"2021-06-02T07:12:17.959431Z","shell.execute_reply":"2021-06-02T07:12:17.971748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nif len(sample_submission) > 4: COMPUTE_CV = False\nif COMPUTE_CV: \n    print('this submission notebook will compute CV score but commit notebook will not')\nelse:\n    print('this submission notebook will only be used to submit result')","metadata":{"papermill":{"duration":0.03378,"end_time":"2021-05-22T12:29:01.603885","exception":false,"start_time":"2021-05-22T12:29:01.570105","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:17.976183Z","iopub.execute_input":"2021-06-02T07:12:17.977143Z","iopub.status.idle":"2021-06-02T07:12:18.001819Z","shell.execute_reply.started":"2021-06-02T07:12:17.976775Z","shell.execute_reply":"2021-06-02T07:12:18.000399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"papermill":{"duration":0.017132,"end_time":"2021-05-22T12:29:01.637825","exception":false,"start_time":"2021-05-22T12:29:01.620693","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)\n\nif COMPUTE_CV: \n    sample_submission = train\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n    test_files_path = paper_test_folder\nelse:\n    sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n    test_files_path = paper_test_folder\n    \nadnl_govt_labels_path = ADNL_GOVT_LABELS_PATH","metadata":{"papermill":{"duration":0.127731,"end_time":"2021-05-22T12:29:01.782376","exception":false,"start_time":"2021-05-22T12:29:01.654645","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:18.006112Z","iopub.execute_input":"2021-06-02T07:12:18.006486Z","iopub.status.idle":"2021-06-02T07:12:18.172174Z","shell.execute_reply.started":"2021-06-02T07:12:18.006455Z","shell.execute_reply":"2021-06-02T07:12:18.170864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(sample_submission['Id']):\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"papermill":{"duration":0.07725,"end_time":"2021-05-22T12:29:01.877146","exception":false,"start_time":"2021-05-22T12:29:01.799896","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:18.174025Z","iopub.execute_input":"2021-06-02T07:12:18.174357Z","iopub.status.idle":"2021-06-02T07:12:18.248735Z","shell.execute_reply.started":"2021-06-02T07:12:18.174325Z","shell.execute_reply":"2021-06-02T07:12:18.247494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal Matching","metadata":{"papermill":{"duration":0.017374,"end_time":"2021-05-22T12:29:01.912536","exception":false,"start_time":"2021-05-22T12:29:01.895162","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Auxiliary Functions","metadata":{"papermill":{"duration":0.01705,"end_time":"2021-05-22T12:29:01.947013","exception":false,"start_time":"2021-05-22T12:29:01.929963","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\n\ndef read_json_pub(filename, train_data_path=train_files_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    \n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"papermill":{"duration":0.030854,"end_time":"2021-05-22T12:29:01.99518","exception":false,"start_time":"2021-05-22T12:29:01.964326","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:18.25063Z","iopub.execute_input":"2021-06-02T07:12:18.251105Z","iopub.status.idle":"2021-06-02T07:12:18.265628Z","shell.execute_reply.started":"2021-06-02T07:12:18.251055Z","shell.execute_reply":"2021-06-02T07:12:18.263975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ken Matching","metadata":{"papermill":{"duration":0.017137,"end_time":"2021-05-22T12:29:02.029652","exception":false,"start_time":"2021-05-22T12:29:02.012515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MLM_ONLY:\n    adnl_govt_labels = pd.read_csv(adnl_govt_labels_path)\n\n    literal_preds = []\n    to_append = []\n    for index, row in tqdm(sample_submission.iterrows()):\n        to_append = [row['Id'],'']\n        large_string = str(read_json_pub(row['Id'], test_files_path))\n        clean_string = text_cleaning(large_string)\n        for index, row2 in adnl_govt_labels.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n                    to_append[1] = to_append[1] + '|' + clean_text(query_string)\n                if to_append[1] == '':\n                    to_append[1] = clean_text(query_string)\n        literal_preds.append(*to_append[1:])","metadata":{"papermill":{"duration":1.075443,"end_time":"2021-05-22T12:29:03.122654","exception":false,"start_time":"2021-05-22T12:29:02.047211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:18.268049Z","iopub.execute_input":"2021-06-02T07:12:18.268726Z","iopub.status.idle":"2021-06-02T07:12:18.533203Z","shell.execute_reply.started":"2021-06-02T07:12:18.268668Z","shell.execute_reply":"2021-06-02T07:12:18.531968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{"papermill":{"duration":0.017958,"end_time":"2021-05-22T12:29:03.158995","exception":false,"start_time":"2021-05-22T12:29:03.141037","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Load model and tokenizer","metadata":{"papermill":{"duration":0.018132,"end_time":"2021-05-22T12:29:03.195411","exception":false,"start_time":"2021-05-22T12:29:03.177279","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MATCH_ONLY:\n    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\n    model = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n\n    mlm = pipeline(\n        'fill-mask', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n    )","metadata":{"papermill":{"duration":13.676262,"end_time":"2021-05-22T12:29:16.889641","exception":false,"start_time":"2021-05-22T12:29:03.213379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:18.535041Z","iopub.execute_input":"2021-06-02T07:12:18.535397Z","iopub.status.idle":"2021-06-02T07:12:27.971278Z","shell.execute_reply.started":"2021-06-02T07:12:18.535361Z","shell.execute_reply":"2021-06-02T07:12:27.969817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary functions","metadata":{"papermill":{"duration":0.018113,"end_time":"2021-05-22T12:29:16.927272","exception":false,"start_time":"2021-05-22T12:29:16.909159","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"papermill":{"duration":0.034379,"end_time":"2021-05-22T12:29:16.979763","exception":false,"start_time":"2021-05-22T12:29:16.945384","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:27.973941Z","iopub.execute_input":"2021-06-02T07:12:27.974493Z","iopub.status.idle":"2021-06-02T07:12:27.992882Z","shell.execute_reply.started":"2021-06-02T07:12:27.97443Z","shell.execute_reply":"2021-06-02T07:12:27.991614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform","metadata":{"papermill":{"duration":0.018008,"end_time":"2021-05-22T12:29:17.015855","exception":false,"start_time":"2021-05-22T12:29:16.997847","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MATCH_ONLY:\n    mask = mlm.tokenizer.mask_token\n    all_test_data = []\n    \n    for paper_id in tqdm(sample_submission['Id']):\n        # load paper\n        paper = papers[paper_id]\n\n        # extract sentences\n        sentences = set([clean_paper_sentence(sentence) for section in paper \n                         for sentence in section['text'].split('.')\n                        ])\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 1]\n        sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n\n        # mask\n        test_data = []\n        for sentence in sentences:\n            for phrase_start, phrase_end in find_mask_candidates(sentence):\n                dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n                test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n\n        all_test_data.append(test_data)","metadata":{"papermill":{"duration":0.117261,"end_time":"2021-05-22T12:29:17.151603","exception":false,"start_time":"2021-05-22T12:29:17.034342","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:27.994665Z","iopub.execute_input":"2021-06-02T07:12:27.995122Z","iopub.status.idle":"2021-06-02T07:12:28.127605Z","shell.execute_reply.started":"2021-06-02T07:12:27.995069Z","shell.execute_reply":"2021-06-02T07:12:28.126273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{"papermill":{"duration":0.018808,"end_time":"2021-05-22T12:29:17.189781","exception":false,"start_time":"2021-05-22T12:29:17.170973","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MATCH_ONLY:\n    pred_mlm_labels = []\n\n    for test_data in tqdm(all_test_data):\n        pred_bag = set()\n\n        if len(test_data):\n            texts, phrases = list(zip(*test_data))\n            mlm_pred = []\n            for p_id in range(0, len(texts), PREDICT_BATCH):\n                batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n                batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n\n                if len(batch_texts) == 1:\n                    batch_pred = [batch_pred]\n\n                mlm_pred.extend(batch_pred)\n\n            for (result1, result2), phrase in zip(mlm_pred, phrases):\n                if (result1['score'] > result2['score']*2 and result1['token_str'] == DATASET_SYMBOL) or\\\n                   (result2['score'] > result1['score']*2 and result2['token_str'] == NONDATA_SYMBOL):\n                    pred_bag.add(clean_text(phrase))\n\n        # filter labels by jaccard score \n        filtered_labels = []\n\n        for label in sorted(pred_bag, key=len, reverse=True):\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n                filtered_labels.append(label)\n\n        pred_mlm_labels.append('|'.join(filtered_labels))\n    \n    pred_mlm_labels[:5]","metadata":{"papermill":{"duration":3.013978,"end_time":"2021-05-22T12:29:20.222921","exception":false,"start_time":"2021-05-22T12:29:17.208943","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:12:28.129628Z","iopub.execute_input":"2021-06-02T07:12:28.130115Z","iopub.status.idle":"2021-06-02T07:13:00.97546Z","shell.execute_reply.started":"2021-06-02T07:12:28.130062Z","shell.execute_reply":"2021-06-02T07:13:00.974564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate Final Predictions","metadata":{"papermill":{"duration":0.019411,"end_time":"2021-05-22T12:29:20.26201","exception":false,"start_time":"2021-05-22T12:29:20.242599","status":"completed"},"tags":[]}},{"cell_type":"code","source":"final_predictions = []\n\nif MATCH_ONLY:\n    final_predictions = literal_preds\nelif MLM_ONLY:\n    final_predictions = pred_mlm_labels\nelse:    \n    for literal_match, mlm_pred in zip(literal_preds, pred_mlm_labels):\n        if literal_match:\n            final_predictions.append(literal_match)\n        else:\n            final_predictions.append(mlm_pred)\n\nsample_submission['PredictionString'] = final_predictions","metadata":{"papermill":{"duration":0.030549,"end_time":"2021-05-22T12:29:20.312145","exception":false,"start_time":"2021-05-22T12:29:20.281596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:13:00.976937Z","iopub.execute_input":"2021-06-02T07:13:00.977263Z","iopub.status.idle":"2021-06-02T07:13:00.986151Z","shell.execute_reply.started":"2021-06-02T07:13:00.977229Z","shell.execute_reply":"2021-06-02T07:13:00.984872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\n\nsample_submission.head()","metadata":{"papermill":{"duration":0.041152,"end_time":"2021-05-22T12:29:20.373001","exception":false,"start_time":"2021-05-22T12:29:20.331849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:13:00.987697Z","iopub.execute_input":"2021-06-02T07:13:00.988058Z","iopub.status.idle":"2021-06-02T07:13:01.029999Z","shell.execute_reply.started":"2021-06-02T07:13:00.988027Z","shell.execute_reply":"2021-06-02T07:13:01.028704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Metric","metadata":{"papermill":{"duration":0.020492,"end_time":"2021-05-22T12:29:20.413643","exception":false,"start_time":"2021-05-22T12:29:20.393151","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/230091\ndef compute_fbeta(y_true: List[List[str]],\n                  y_pred: List[List[str]],\n                  beta: float = 0.5) -> float:\n    \"\"\"Compute the Jaccard-based micro FBeta score.\n\n    References\n    ----------\n    - https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\n    \"\"\"\n\n    def _jaccard_similarity(str1: str, str2: str) -> float:\n        a = set(str1.split()) \n        b = set(str2.split())\n        c = a.intersection(b)\n        return float(len(c)) / (len(a) + len(b) - len(c))\n\n    tp = 0  # true positive\n    fp = 0  # false positive\n    fn = 0  # false negative\n    for ground_truth_list, predicted_string_list in zip(y_true, y_pred):\n        predicted_string_list_sorted = sorted(predicted_string_list)\n        for ground_truth in sorted(ground_truth_list):            \n            if len(predicted_string_list_sorted) == 0:\n                fn += 1\n            else:\n                similarity_scores = [\n                    _jaccard_similarity(ground_truth, predicted_string)\n                    for predicted_string in predicted_string_list_sorted\n                ]\n                matched_idx = np.argmax(similarity_scores)\n                if similarity_scores[matched_idx] >= 0.5:\n                    predicted_string_list_sorted.pop(matched_idx)\n                    tp += 1\n                else:\n                    fn += 1\n        fp += len(predicted_string_list_sorted)\n\n    tp *= (1 + beta ** 2)\n    fn *= beta ** 2\n    fbeta_score = tp / (tp + fp + fn)\n    return fbeta_score","metadata":{"papermill":{"duration":0.032115,"end_time":"2021-05-22T12:29:20.466017","exception":false,"start_time":"2021-05-22T12:29:20.433902","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:13:01.031646Z","iopub.execute_input":"2021-06-02T07:13:01.031972Z","iopub.status.idle":"2021-06-02T07:13:01.04451Z","shell.execute_reply.started":"2021-06-02T07:13:01.03194Z","shell.execute_reply":"2021-06-02T07:13:01.042726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_CV:\n    COMPUTE_CV_SCORE = compute_fbeta(sample_submission['cleaned_label'].apply(lambda x: [x]),\\\n                  sample_submission['PredictionString'].apply(lambda x: x.split('|')))\n    print('COMPUTE_CV_SCORE =', COMPUTE_CV_SCORE)\nelse:\n    print(f'COMPUTE_CV = {COMPUTE_CV}')\n    \nprint(f'MATCH_ONLY = {MATCH_ONLY}')\nprint(f'MLM_ONLY = {MLM_ONLY}')\nprint(f'ADNL_GOVT_LABELS_PATH = {ADNL_GOVT_LABELS_PATH}')\nprint(f'PRETRAINED_PATH = {PRETRAINED_PATH}')\nprint(f'TOKENIZER_PATH = {TOKENIZER_PATH}')","metadata":{"papermill":{"duration":0.03024,"end_time":"2021-05-22T12:29:20.516417","exception":false,"start_time":"2021-05-22T12:29:20.486177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-02T07:13:01.046365Z","iopub.execute_input":"2021-06-02T07:13:01.046796Z","iopub.status.idle":"2021-06-02T07:13:01.067896Z","shell.execute_reply.started":"2021-06-02T07:13:01.046757Z","shell.execute_reply":"2021-06-02T07:13:01.066097Z"},"trusted":true},"execution_count":null,"outputs":[]}]}