{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"COMPUTE_CV = False\n\nHOW = 'MLM_ONLY' # 'MATCH_ONLY', 'MLM_ONLY', 'UNION_MERGE', 'LAM', 'ALL_BLENDED_PP', 'ORIGINAL'\nADNL_GOVT_LABELS_PATH = '../input/bigger-govt-dataset-list/data_set_800.csv'\nMATCHING_DATA = 'original' # 'original', 'drop_duplicates', 'text_cleaning'\nPRED_TH = 2.0\nFL_TH = 1.0 # 0.75\nLAM_FL_TH = 0.5\n\nPOST_PROCESSING = False; PP_TH = 0.5\nSEED = 42\n\nif HOW != 'MATCH_ONLY':\n    MODEL_PATH_PREFIX = '../input/coleridge-bert-mlm-external-pseudo-labels'\n    MLM_PRETRAINED_PATH = 'mlm-model'\n    TOKENIZER = 'model_tokenizer'\n    \n    LENGTH = 1\n    MAX_LENGTH = 64\n    OVERLAP = 20\n\n    PREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\n    DATASET_SYMBOL = '$' # this symbol represents a dataset name\n    NONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:29.049473Z","iopub.execute_input":"2021-05-31T04:58:29.049884Z","iopub.status.idle":"2021-05-31T04:58:29.056683Z","shell.execute_reply.started":"2021-05-31T04:58:29.049852Z","shell.execute_reply":"2021-05-31T04:58:29.055702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\n\nimport os, sys\nimport re\nimport json\nimport random\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.autonotebook import tqdm\nfrom datasets import load_dataset\nimport torch\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\n\nfrom IPython.display import clear_output\n\n\nclear_output()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-31T04:37:33.117476Z","iopub.execute_input":"2021-05-31T04:37:33.117892Z","iopub.status.idle":"2021-05-31T04:39:17.792945Z","shell.execute_reply.started":"2021-05-31T04:37:33.117863Z","shell.execute_reply":"2021-05-31T04:39:17.79177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://huggingface.co/transformers/_modules/transformers/trainer_utils.html\ndef set_seed(seed: int):\n    \"\"\"\n    Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch`` and/or ``tf`` (if\n    installed).\n\n    Args:\n        seed (:obj:`int`): The seed to set.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # ^^ safe to call this function even if cuda is not available\n    \n    print(f'Setted Pipeline SEED = {SEED}')\n\n\nset_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:17.794778Z","iopub.execute_input":"2021-05-31T04:39:17.795212Z","iopub.status.idle":"2021-05-31T04:39:17.80893Z","shell.execute_reply.started":"2021-05-31T04:39:17.795175Z","shell.execute_reply":"2021-05-31T04:39:17.807663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nif len(sample_submission) > 4: COMPUTE_CV = False\nif COMPUTE_CV: \n    print('this submission notebook will compute CV score but commit notebook will not')\nelse:\n    print('this submission notebook will only be used to submit result')","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:17.810746Z","iopub.execute_input":"2021-05-31T04:39:17.811424Z","iopub.status.idle":"2021-05-31T04:39:17.845458Z","shell.execute_reply.started":"2021-05-31T04:39:17.811368Z","shell.execute_reply":"2021-05-31T04:39:17.843923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)\n\nif COMPUTE_CV: \n    sample_submission = train\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n    test_files_path = paper_test_folder\nelse:\n    sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n    test_files_path = paper_test_folder\n    \nadnl_govt_labels_path = ADNL_GOVT_LABELS_PATH","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:17.849722Z","iopub.execute_input":"2021-05-31T04:39:17.85037Z","iopub.status.idle":"2021-05-31T04:39:18.038819Z","shell.execute_reply.started":"2021-05-31T04:39:17.850323Z","shell.execute_reply":"2021-05-31T04:39:18.037622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(sample_submission['Id']):\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:18.040753Z","iopub.execute_input":"2021-05-31T04:39:18.041137Z","iopub.status.idle":"2021-05-31T04:39:18.136172Z","shell.execute_reply.started":"2021-05-31T04:39:18.0411Z","shell.execute_reply":"2021-05-31T04:39:18.134976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal Matching","metadata":{}},{"cell_type":"markdown","source":"## Auxiliary Functions","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\n\ndef read_json_pub(filename, train_data_path=train_files_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    \n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:18.137927Z","iopub.execute_input":"2021-05-31T04:39:18.138547Z","iopub.status.idle":"2021-05-31T04:39:18.153248Z","shell.execute_reply.started":"2021-05-31T04:39:18.138499Z","shell.execute_reply":"2021-05-31T04:39:18.15211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ken Matching","metadata":{}},{"cell_type":"code","source":"if HOW != 'MLM_ONLY':\n    adnl_govt_labels = pd.read_csv(adnl_govt_labels_path)\n    print('adnl_govt_labels size =', len(adnl_govt_labels))\n    \n    if MATCHING_DATA in ['drop_duplicates', 'text_cleaning']:\n        adnl_govt_labels = adnl_govt_labels.drop_duplicates()\n        print('adnl_govt_labels size after drop_duplicates =', len(adnl_govt_labels))\n        \n        if MATCHING_DATA == 'text_cleaning':\n            adnl_govt_labels['title'] = adnl_govt_labels['title'].apply(text_cleaning)\n            adnl_govt_labels = adnl_govt_labels.drop_duplicates()\n            print('adnl_govt_labels size after text_cleaning =', len(adnl_govt_labels))\n\n    \n    literal_preds = []\n    to_append = []\n    for index, row in tqdm(sample_submission.iterrows()):\n        to_append = [row['Id'],'']\n        large_string = str(read_json_pub(row['Id'], test_files_path))\n        clean_string = text_cleaning(large_string)\n        for index, row2 in adnl_govt_labels.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n                    to_append[1] = to_append[1] + '|' + clean_text(query_string)\n                if to_append[1] == '':\n                    to_append[1] = clean_text(query_string)\n        literal_preds.append(*to_append[1:])\n\n    print(literal_preds[:4])\n\nelse: print('Matching is not used.')","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:18.154854Z","iopub.execute_input":"2021-05-31T04:39:18.155566Z","iopub.status.idle":"2021-05-31T04:39:19.521701Z","shell.execute_reply.started":"2021-05-31T04:39:18.155508Z","shell.execute_reply":"2021-05-31T04:39:19.520272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{}},{"cell_type":"markdown","source":"## Load model and tokenizer","metadata":{}},{"cell_type":"code","source":"if HOW != 'MATCH_ONLY':\n    TOKENIZER_PATH = os.path.join(MODEL_PATH_PREFIX, TOKENIZER)\n    PRETRAINED_PATH = os.path.join(MODEL_PATH_PREFIX, MLM_PRETRAINED_PATH)\n    \n    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\n    model = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n\n    mlm = pipeline(\n        'fill-mask', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n    )","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:19.523581Z","iopub.execute_input":"2021-05-31T04:39:19.524051Z","iopub.status.idle":"2021-05-31T04:39:30.957131Z","shell.execute_reply.started":"2021-05-31T04:39:19.524005Z","shell.execute_reply":"2021-05-31T04:39:30.955859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary functions","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:30.958909Z","iopub.execute_input":"2021-05-31T04:39:30.959439Z","iopub.status.idle":"2021-05-31T04:39:30.977189Z","shell.execute_reply.started":"2021-05-31T04:39:30.959383Z","shell.execute_reply":"2021-05-31T04:39:30.975785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform","metadata":{}},{"cell_type":"code","source":"if HOW != 'MATCH_ONLY':\n    mask = mlm.tokenizer.mask_token\n    all_test_data = []\n    \n    for paper_id in tqdm(sample_submission['Id']):\n        # load paper\n        paper = papers[paper_id]\n\n        # extract sentences\n        sentences = set([clean_paper_sentence(sentence) for section in paper \n                         for sentence in section['text'].split('.')\n                        ])\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > LENGTH] # only accept sentences with length > 10 chars\n        sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n\n        # mask\n        test_data = []\n        for sentence in sentences:\n            for phrase_start, phrase_end in find_mask_candidates(sentence):\n                dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n                test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n\n        all_test_data.append(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:30.978892Z","iopub.execute_input":"2021-05-31T04:39:30.979375Z","iopub.status.idle":"2021-05-31T04:39:31.102394Z","shell.execute_reply.started":"2021-05-31T04:39:30.979325Z","shell.execute_reply":"2021-05-31T04:39:31.101028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"# test_data = all_test_data[0]\n# pred_bag = set()\n# if len(test_data):\n#     texts, phrases = list(zip(*test_data))\n#     mlm_pred = []\n#     for p_id in range(0, len(texts), PREDICT_BATCH):\n#         batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n#         batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n\n#         if len(batch_texts) == 1:\n#             batch_pred = [batch_pred]\n\n#         mlm_pred.extend(batch_pred)\n\n#     for find_one_th in np.arange(FIND_ONE_TH, 1.0, -0.05): # abandoned this one\n#         if len(pred_bag) == 0:\n#             score1_temp = []; score2_temp = []\n#             phrase_temp = []\n#             token_str1 = []; token_str2 = []\n#             for (result1, result2), phrase in zip(mlm_pred, phrases):\n#                 score1_temp.append(result1['score']); score2_temp.append(result2['score'])\n#                 phrase_temp.append(phrase)\n#                 token_str1.append(result1['token_str']); token_str2.append(result2['token_str'])\n#                 if (result1['score'] > result2['score'] * find_one_th and result1['token_str'] == DATASET_SYMBOL) or\\\n#                    (result2['score'] > result1['score'] * find_one_th and result2['token_str'] == NONDATA_SYMBOL):\n#                     pred_bag.add(clean_text(phrase))\n                    \n            \n            \n# #             if len(pred_bag) == 0:\n#             if True:\n#                 score_df = pd.DataFrame({'score1': score1_temp, 'score2': score2_temp,\n#                                         'phrase': phrase_temp,\n#                                         'token_str1': token_str1, 'token_str2': token_str2})\n#                 score_df['score_diff'] = abs(score_df.score1 - score_df.score2)\n#                 row = score_df['score_diff'].argmax()\n#                 pred_bag.add(score_df['phrase'][row])\n# pred_bag","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:39:31.104225Z","iopub.execute_input":"2021-05-31T04:39:31.104692Z","iopub.status.idle":"2021-05-31T04:39:31.110682Z","shell.execute_reply.started":"2021-05-31T04:39:31.104645Z","shell.execute_reply":"2021-05-31T04:39:31.109325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if HOW != 'MATCH_ONLY':\n    pred_mlm_labels = []\n\n    for test_data in tqdm(all_test_data):\n        pred_bag = set()\n\n        if len(test_data):\n            texts, phrases = list(zip(*test_data))\n            mlm_pred = []\n            for p_id in range(0, len(texts), PREDICT_BATCH):\n                batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n                batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n                \n                if len(batch_texts) == 1:\n                    batch_pred = [batch_pred]\n                    \n                mlm_pred.extend(batch_pred)\n\n            for pred_th in np.arange(PRED_TH, 1.0, -0.05): # find-more PP\n                if len(pred_bag) == 0:\n                    for (result1, result2), phrase in zip(mlm_pred, phrases):\n                        if (result1['score'] > result2['score'] * pred_th and result1['token_str'] == DATASET_SYMBOL) or\\\n                           (result2['score'] > result1['score'] * pred_th and result2['token_str'] == NONDATA_SYMBOL):\n                            pred_bag.add(clean_text(phrase))\n                else: break\n\n        # filter labels by jaccard score \n        filtered_labels = []\n        for label in sorted(pred_bag, key=len, reverse=True): # long to short so that we keep the potential best\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < FL_TH for got_label in filtered_labels):\n                filtered_labels.append(label)\n\n        pred_mlm_labels.append('|'.join(filtered_labels))\n        \n    print(f'pred_mlm_labels[:4] w/ PT{PRED_TH} FL{FL_TH}: \\n{pred_mlm_labels[:4]}')\n        \nelse: print('MLM is not used.')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-05-31T04:39:31.112352Z","iopub.execute_input":"2021-05-31T04:39:31.1127Z","iopub.status.idle":"2021-05-31T04:40:05.482949Z","shell.execute_reply.started":"2021-05-31T04:39:31.112668Z","shell.execute_reply":"2021-05-31T04:40:05.481557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Predictions","metadata":{}},{"cell_type":"code","source":"final_predictions = []\n\n\nif HOW == 'MATCH_ONLY':\n    final_predictions = literal_preds\n    \nelif HOW == 'MLM_ONLY':\n    final_predictions = pred_mlm_labels\n\nelif HOW == 'UNION_MERGE':\n    for i in range(len(literal_preds)):\n        pred_naive = literal_preds[i].split('|')\n        pred_model = pred_mlm_labels[i].split('|')\n        pred_model_kept = []\n        for pred_m in pred_model:\n            kept = True\n            for pred_n in pred_naive:\n                if pred_m in pred_n or pred_n in pred_m:\n                    kept = False\n            if kept:\n                pred_model_kept.append(pred_m)\n        final_predictions.append(\"|\".join(pred_naive + pred_model_kept))\n        \nelif HOW == 'LAM':\n    for pred_match, perd_mlm in tqdm(zip(literal_preds, pred_mlm_labels)):\n        if pred_match:\n            labels = pred_match.split('|')\n            \n            # literal_preds + pred_mlm_labels\n            if perd_mlm:\n                filtered_labels = labels\n                labels_mlm = perd_mlm.split('|')\n                for label_mlm in labels_mlm:\n                    if all(jaccard_similarity(label_mlm, got_label) < LAM_FL_TH for got_label in labels):\n                        filtered_labels.append(label_mlm)\n                        \n            # literal_preds\n            else: filtered_labels = labels\n                \n        # pred_mlm_labels\n        elif perd_mlm:\n            filtered_labels = perd_mlm.split('|')\n        \n        # ''\n        else:\n            filtered_labels = []\n            \n        final_predictions.append('|'.join(filtered_labels))\n    print(f'final_predictions[:4] w/ LAM_FL_TH{LAM_FL_TH}:')\n        \nelif HOW == 'ALL_BLENDED_PP':\n    for pred_match, perd_mlm in tqdm(zip(literal_preds, pred_mlm_labels)):\n        if pred_match and perd_mlm:\n            labels = pred_match + '|' + perd_mlm\n        elif pred_match:\n            labels = pred_match\n        elif perd_mlm:\n            labels = pred_match # bug to external-pseudo v1 length1 FL0.5 ABPP | 0.482\n        else:\n            labels = ''\n        labels = labels.split('|')\n        filtered_labels = []\n        for label in sorted(labels, key=len, reverse=True):\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < FL_TH for got_label in filtered_labels):\n                filtered_labels.append(label)\n        final_predictions.append('|'.join(filtered_labels))\n    print(f'final_predictions[:4] w/ FL{FL_TH}:')\n        \nelif HOW == 'ORIGINAL':    \n    for literal_match, mlm_pred in zip(literal_preds, pred_mlm_labels):\n        if literal_match:\n            final_predictions.append(literal_match)\n        else:\n            final_predictions.append(mlm_pred)\n\n\nfinal_predictions[:4]","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:37.387038Z","iopub.execute_input":"2021-05-31T04:58:37.38775Z","iopub.status.idle":"2021-05-31T04:58:37.441497Z","shell.execute_reply.started":"2021-05-31T04:58:37.387689Z","shell.execute_reply":"2021-05-31T04:58:37.439952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[What is your best score without string matching?](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/232964)","metadata":{}},{"cell_type":"code","source":"if POST_PROCESSING:\n    temp_1 = [x.lower() for x in train['dataset_label'].unique()]\n    temp_2 = [x.lower() for x in train['dataset_title'].unique()]\n    temp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n    existing_labels = list(set(temp_1 + temp_2 + temp_3))\n    \n    final_predictions_temp = []\n    for labels in final_predictions:\n        labels = labels.split('|')\n        final_predictions_temp_temp = []\n        for label in labels:\n            if all(jaccard_similarity(label, existing_label) < PP_TH for existing_label in existing_labels):\n                final_predictions_temp_temp.append(label)\n        final_predictions_temp.append('|'.join(final_predictions_temp_temp))\n    final_predictions = final_predictions_temp\n    \n    print(f'final_predictions[:4] w/ PP{str(PP_TH)[1:]}:')\n    \nelse:\n    print('final_predictions[:4] w/o PP:')\n    \n\nfinal_predictions[:4]","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:37.445595Z","iopub.execute_input":"2021-05-31T04:58:37.445971Z","iopub.status.idle":"2021-05-31T04:58:37.460752Z","shell.execute_reply.started":"2021-05-31T04:58:37.445941Z","shell.execute_reply":"2021-05-31T04:58:37.459166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\n\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:37.462926Z","iopub.execute_input":"2021-05-31T04:58:37.463496Z","iopub.status.idle":"2021-05-31T04:58:37.488304Z","shell.execute_reply.started":"2021-05-31T04:58:37.463434Z","shell.execute_reply":"2021-05-31T04:58:37.486755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Metric","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/230091\ndef compute_fbeta(y_true: List[List[str]],\n                  y_pred: List[List[str]],\n                  beta: float = 0.5) -> float:\n    \"\"\"Compute the Jaccard-based micro FBeta score.\n\n    References\n    ----------\n    - https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\n    \"\"\"\n\n    def _jaccard_similarity(str1: str, str2: str) -> float:\n        a = set(str1.split()) \n        b = set(str2.split())\n        c = a.intersection(b)\n        return float(len(c)) / (len(a) + len(b) - len(c))\n\n    tp = 0  # true positive\n    fp = 0  # false positive\n    fn = 0  # false negative\n    for ground_truth_list, predicted_string_list in zip(y_true, y_pred):\n        predicted_string_list_sorted = sorted(predicted_string_list)\n        for ground_truth in sorted(ground_truth_list):            \n            if len(predicted_string_list_sorted) == 0:\n                fn += 1\n            else:\n                similarity_scores = [\n                    _jaccard_similarity(ground_truth, predicted_string)\n                    for predicted_string in predicted_string_list_sorted\n                ]\n                matched_idx = np.argmax(similarity_scores)\n                if similarity_scores[matched_idx] >= 0.5:\n                    predicted_string_list_sorted.pop(matched_idx)\n                    tp += 1\n                else:\n                    fn += 1\n        fp += len(predicted_string_list_sorted)\n\n    tp *= (1 + beta ** 2)\n    fn *= beta ** 2\n    fbeta_score = tp / (tp + fp + fn)\n    return fbeta_score","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:37.489858Z","iopub.execute_input":"2021-05-31T04:58:37.49015Z","iopub.status.idle":"2021-05-31T04:58:37.5024Z","shell.execute_reply.started":"2021-05-31T04:58:37.490123Z","shell.execute_reply":"2021-05-31T04:58:37.500914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_CV:\n    COMPUTE_CV_SCORE = compute_fbeta(sample_submission['cleaned_label'].apply(lambda x: [x]),\\\n                  sample_submission['PredictionString'].apply(lambda x: x.split('|')))\n    print('COMPUTE_CV_SCORE =', COMPUTE_CV_SCORE)\nelse:\n    print(f'COMPUTE_CV = {COMPUTE_CV}')\n    \nprint(f'HOW = {HOW}')\nprint(f'ADNL_GOVT_LABELS_PATH = {ADNL_GOVT_LABELS_PATH}')\nprint(f'MATCHING_DATA = {MATCHING_DATA}')\nprint(f'PRED_TH = {PRED_TH}')\nprint(f'FL_TH = {FL_TH}')\nif HOW == 'LAM': print(f'LAM_FL_TH = {LAM_FL_TH}')\n\nprint(f'POST_PROCESSING = {POST_PROCESSING}')\nif POST_PROCESSING: print(f'PP_TH = {PP_TH}')\n\nprint(f'PRETRAINED_PATH = {PRETRAINED_PATH}')\nprint(f'TOKENIZER_PATH = {TOKENIZER_PATH}')\nprint(f'LENGTH = {LENGTH}')","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:58:37.504458Z","iopub.execute_input":"2021-05-31T04:58:37.504784Z","iopub.status.idle":"2021-05-31T04:58:37.522737Z","shell.execute_reply.started":"2021-05-31T04:58:37.504743Z","shell.execute_reply":"2021-05-31T04:58:37.521462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|   | CV | LB |\n| --- | --- | --- |\n| v1 length10 PLv1 only |   | 0.001 |\n| v2 length8 |   | 0.575 |\n| v2 length8 UM |   | 0.554 |\n| v2 length8 only |   | 0.170 |\n| v2 length8 36 og |   | 0.573 |\n| v2 length8 36 add |   | 0.574 |\n| v2 length8 48 |   | 0.574 |\n| v3 length5 |   | 0.572 |\n| v3 length5 36 |   | 0.573 |\n| v4 length1 |   | 0.574 |\n| v4 length1 36 |   | 0.573 |\n| **v4 length1 48** | **0.514** | **0.575** |\n| v4 length1 48 PP.8 |   |   |\n| v4 length1 48 only |   | 0.106 |\n| v4 length1 48 after drop_duplicates |   | 0.575 |\n| v4 length1 48 after text_cleaning |   | 0.573 |\n| v4 length1 48 UM |   | 0.561 |\n| v4 length1 48 2000 |   | 0.532 |\n| v4 length1 48 2000 text_cleaning |   | 0.531 |\n| v4 length1 48 2000 UM |   | 0.521 |\n| v4 length1 48 4000 UM |   | 0.486 |\n| v4 length1 48 4000 |   | 0.496 |\n| v4 length1 48 26897 |   | 0.244 |\n| v4 length1 60 |   | 0.575 |\n| NER w/ External_Datasets_Matching |   | 0.573 |\n| NER v1 length1 |   | 0.573 |\n| NER v1 length1 45 |   | 0.573 |\n| external-data v1 length1 |   | 0.573 |\n| external-data v1 length1 only |   | 0.118 |\n| external-data v2 length1 |   | 0.574 |\n| external-data v2 length1 only |   | 0.207 |\n| external-data v2 length1 84 |   | 0.573 |\n| external-data v2 length1 72 |   | 0.574 |\n| external-data v2 length1 60 |   | 0.574 |\n| external-data v2 length1 48 |   | 0.570 |\n| external-data v2 length1 36 |   | 0.572 |\n| external-pseudo v1 length1 |   | 0.572 |\n| external-pseudo v1 length1 FOPP FL1.0 |   | 0.571 |\n| external-pseudo v1 length1 FOPP FL1.0 LAM0.5 |   |   |\n| external-pseudo v1 length1 only FL1.0 |   | 0.215 |\n| external-pseudo v3 length1 (FL1.0) only FOPP FL1.0 |   | 0.216 |\n| external-pseudo v4 length8 only FOPP FL1.0 |   | 0.196 |\n| **external-pseudo v1 length1 only FOPP FL1.0** |   | **0.227** |\n| external-pseudo v5 length1 only FOPP FL1.0 |   |   |\n| external-pseudo v1 length1 only PT2.5 FOPP FL1.0 |   | 0.215 |\n| external-pseudo v2 length1 (FL0.5) only FOPP FL1.0 |   | 0.225 |\n| external-pseudo v1 length1 only |   | 0.213 |\n| external-pseudo v1 length1 only FOPP |   |   |\n| external-pseudo v1 length1 only FL0.5 |   | 0.206|\n| external-pseudo v1 length1 FL0.5 |   | 0.572 |\n| external-pseudo v1 length1 FL0.5 ABPP(bug) |   | 0.482 |\n| external-pseudo v1 length1 only PP.5 |   |   |","metadata":{}}]}