{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport json\nimport re\nimport numpy as np\nimport string\nfrom functools import partial\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\n\n\nfrom transformers import TFElectraForPreTraining, ElectraTokenizerFast\n\nimport tensorflow as tf\n\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.layers import Dense, Input, LSTM, Bidirectional, SpatialDropout1D\nfrom tensorflow.keras.models import Model\nimport tensorflow.keras.backend as K\n\nfrom tensorflow_addons.text.crf import crf_log_likelihood\nfrom tensorflow_addons.layers.crf import CRF","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T14:26:34.760825Z","iopub.execute_input":"2021-06-22T14:26:34.761391Z","iopub.status.idle":"2021-06-22T14:26:40.345783Z","shell.execute_reply.started":"2021-06-22T14:26:34.761290Z","shell.execute_reply":"2021-06-22T14:26:40.344824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '../input/coleridgeinitiative-show-us-the-data'\n\ntest_dir = os.path.join(BASE_DIR, 'test')\n\nsample_submission_path = os.path.join(BASE_DIR, 'sample_submission.csv')\nsample_df = pd.read_csv(sample_submission_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.349761Z","iopub.execute_input":"2021-06-22T14:26:40.350023Z","iopub.status.idle":"2021-06-22T14:26:40.364515Z","shell.execute_reply.started":"2021-06-22T14:26:40.349990Z","shell.execute_reply":"2021-06-22T14:26:40.363728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"url_regex = re.compile(\"https?://[\\w!\\?/\\+\\-_~=;\\.,\\*&@#\\$%\\(\\)'\\[\\]]+[\\w!\\?/\\+\\-_~=\\*&@#\\$%']\")\nwww_regex = re.compile(\"www\\.[\\w!\\?/\\+\\-_~=;\\.,\\*&@#\\$%\\(\\)'\\[\\]]+[\\w!\\?/\\+\\-_~=\\*&@#\\$%']\")\ndef get_article(filename, dir_path=test_dir):\n    json_path = os.path.join(test_dir, (filename+'.json'))\n    contents = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            section_title = data['section_title']\n            section_text= data['text']\n            if len(section_text) >= len(section_title):\n                contents.append(section_text)\n            else:\n                contents.append(section_title)\n    all_contents = ' '.join(contents)\n\n    return www_regex.sub('', url_regex.sub('', all_contents))\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef jaccard_similarity(s1, s2):\n    a = set(s1.lower().split()) \n    b = set(s2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.368230Z","iopub.execute_input":"2021-06-22T14:26:40.368479Z","iopub.status.idle":"2021-06-22T14:26:40.379485Z","shell.execute_reply.started":"2021-06-22T14:26:40.368455Z","shell.execute_reply":"2021-06-22T14:26:40.378702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsample_df['text'] = sample_df['Id'].progress_apply(get_article)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.380725Z","iopub.execute_input":"2021-06-22T14:26:40.381096Z","iopub.status.idle":"2021-06-22T14:26:40.467630Z","shell.execute_reply.started":"2021-06-22T14:26:40.381060Z","shell.execute_reply":"2021-06-22T14:26:40.466871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = os.path.join(BASE_DIR, 'train.csv')\ntrain_df = pd.read_csv(train_path)\n\nbracket_regex = re.compile(\"\\(.+\\)\")\n\ntrain_path = os.path.join(BASE_DIR, 'train.csv')\ntrain_tmp = pd.read_csv(train_path)\ntemp_1 = { \n    bracket_regex.sub('', x).lower().strip() \n    if len(x.split()) > 1 else bracket_regex.sub('', x).lower().strip() + ' ' \n    for x in train_df['dataset_label'].unique()\n}\ntemp_2 = { \n    bracket_regex.sub('', x).lower().strip() \n    if len(x.split()) > 1 else bracket_regex.sub('', x).lower().strip() + ' ' \n    for x in train_df['dataset_title'].unique()\n}\nexisting_labels ={ label for label in (temp_1 | temp_2)}\nexisting_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.468827Z","iopub.execute_input":"2021-06-22T14:26:40.469206Z","iopub.status.idle":"2021-06-22T14:26:40.650269Z","shell.execute_reply.started":"2021-06-22T14:26:40.469169Z","shell.execute_reply":"2021-06-22T14:26:40.649534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOPIC_WORDS = {\n    'Study', 'Studies', 'Survey', 'Data', 'Progress', 'Consortium', 'Surveillance', 'Assessment', '1972', ' Aging', 'Inventory', 'Atherosclerosis', 'Religious '\n}\nSTOP_WORDS = {\n    'Cooperative', 'Analysis', 'Board', 'Center', 'Climate', 'Report', 'Geodetic', 'Hydrography', 'Initiative', 'Institute', 'Integrated', 'Kindergarten', \n    'Layer', 'Mayo', 'Montreal', 'Panel', 'Questionnaire', 'Adequate', 'Quality', 'Information', 'Harvard ', 'State', 'Scale' 'Transcript',  'Research ', \n    'US ', 'Uniform'\n}\n\nDF_THRES = 20\n\ndef get_additional_labels(extract_results ,existing_labels, \n                          topic_words=TOPIC_WORDS, stop_words=STOP_WORDS, df_thres=DF_THRES):\n    addtional_labels = set()\n    for target, df_value in extract_results.items():\n        if (df_value >= df_thres and\n            any(topic_word in target for topic_word in topic_words) and\n            all(stop_word not in target for stop_word in stop_words)):\n                cleaned_target = clean_text(target)\n                if all(jaccard_similarity(cleaned_target, label) < 0.5 for label in existing_labels):\n                    addtional_labels.add(target.lower().strip())\n    return addtional_labels\n\n\ncleaned_existing_labels = {clean_text(label) for label in existing_labels}\nextract_results_path = '../input/extract-result/extract_results.json'\nwith open(extract_results_path, 'r') as f:\n    extract_results = json.load(f)\n    addtional_sets = get_additional_labels(extract_results, cleaned_existing_labels)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.652678Z","iopub.execute_input":"2021-06-22T14:26:40.653000Z","iopub.status.idle":"2021-06-22T14:26:40.731479Z","shell.execute_reply.started":"2021-06-22T14:26:40.652966Z","shell.execute_reply":"2021-06-22T14:26:40.730708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_bracket_regex = re.compile('(?<=\\().+?(?=\\))')\nabb_extract_results_path = '../input/extract-results-with-abbreviation/extract_results_with_abbreviation.json'\nwith open(abb_extract_results_path, 'r') as f:\n    extract_results = json.load(f)\n    abbreviation_patterns = get_additional_labels(extract_results, cleaned_existing_labels, df_thres=100)\n    abbreviations = set()\n    for abbreviation_pattern in abbreviation_patterns:\n        abbreviation = in_bracket_regex.findall(abbreviation_pattern)[0]\n        if len(abbreviation) > 3:\n            abbreviations.add(f'{abbreviation} ')  \n\naddtional_sets |= abbreviations","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.733204Z","iopub.execute_input":"2021-06-22T14:26:40.733579Z","iopub.status.idle":"2021-06-22T14:26:40.797312Z","shell.execute_reply.started":"2021-06-22T14:26:40.733542Z","shell.execute_reply":"2021-06-22T14:26:40.796479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"addtional_labels = sorted(addtional_sets, key=lambda x: len(x.split()), reverse=True)\naddtional_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.798564Z","iopub.execute_input":"2021-06-22T14:26:40.799080Z","iopub.status.idle":"2021-06-22T14:26:40.807306Z","shell.execute_reply.started":"2021-06-22T14:26:40.799039Z","shell.execute_reply":"2021-06-22T14:26:40.806194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 128\nBATCH_SIZE = 128\nENCODER_DIR = '/kaggle/input/huggingfaceelectra/electra-base-discriminator'","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.808652Z","iopub.execute_input":"2021-06-22T14:26:40.809095Z","iopub.status.idle":"2021-06-22T14:26:40.815487Z","shell.execute_reply.started":"2021-06-22T14:26:40.809044Z","shell.execute_reply":"2021-06-22T14:26:40.814694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = ElectraTokenizerFast.from_pretrained(ENCODER_DIR)\nlabel2id = {\n    tokenizer.pad_token: 0,\n    tokenizer.cls_token: 1,\n    tokenizer.sep_token: 2,\n    'B-DATA': 3,\n    'I-DATA': 4,\n    'O': 5\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.816842Z","iopub.execute_input":"2021-06-22T14:26:40.817579Z","iopub.status.idle":"2021-06-22T14:26:40.885106Z","shell.execute_reply.started":"2021-06-22T14:26:40.817523Z","shell.execute_reply":"2021-06-22T14:26:40.884414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_data(data):\n    if len(data) == 2:\n        return data[0], data[1], None\n    elif len(data) == 3:\n        return data\n    else:\n        raise TypeError(\"Expected data to be a tuple of size 2 or 3.\")\n\nclass ModelWithCRFLoss(tf.keras.Model):\n    \"\"\"Wrapper around the base model for custom training logic.\"\"\"\n\n    def __init__(self, base_model):\n        super().__init__()\n        self.base_model = base_model\n\n    @tf.function\n    def call(self, inputs):\n        return self.base_model(inputs)\n\n    def compute_loss(self, x, y, sample_weight, training=False):\n        y_pred = self(x, training=training)\n        _, potentials, sequence_length, chain_kernel = y_pred\n\n        # we now add the CRF loss:\n        crf_loss = -crf_log_likelihood(potentials, y, sequence_length, chain_kernel)[0]\n\n        if sample_weight is not None:\n            crf_loss = crf_loss * sample_weight\n\n        return tf.reduce_mean(crf_loss), sum(self.losses)\n\n    @tf.function\n    def train_step(self, data):\n        x, y, sample_weight = unpack_data(data)\n\n        with tf.GradientTape() as tape:\n            crf_loss, internal_losses = self.compute_loss(\n                x, y, sample_weight, training=True\n            )\n            total_loss = crf_loss + internal_losses\n\n        gradients = tape.gradient(total_loss, self.trainable_variables)\n        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n\n        return {\"crf_loss\": crf_loss, \"internal_losses\": internal_losses}\n\n    @tf.function\n    def test_step(self, data):\n        x, y, sample_weight = unpack_data(data)\n        crf_loss, internal_losses = self.compute_loss(x, y, sample_weight)\n        return {\"crf_loss\": crf_loss, \"internal_losses\": internal_losses}\n\ndef build_base_model(transformer, num_cls=1, max_len=512):\n    input_ids = Input(shape=(max_len,), dtype=tf.int32, name='input_ids')\n    input_attention_mask = Input(shape=(max_len,), dtype=tf.int32, name='attention_mask')\n    sequence_output = transformer({\n        'input_ids': input_ids, \n        'attention_mask': input_attention_mask\n    }).hidden_states[0]\n    mask = tf.cast(input_attention_mask, tf.bool)\n    sequence_output = SpatialDropout1D(0.1)(sequence_output)\n    sequence_output = Bidirectional(LSTM(256, return_sequences=True), name='bidirectional_lstm')(sequence_output, mask=mask)\n    sequence_output = Dense(num_cls, activation='softmax', name='sequence_output')(sequence_output)\n    out = CRF(num_cls, name='crf_output')(sequence_output, mask=mask)\n    model = Model(inputs=[input_ids, input_attention_mask], outputs=out)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.886341Z","iopub.execute_input":"2021-06-22T14:26:40.886719Z","iopub.status.idle":"2021-06-22T14:26:40.901322Z","shell.execute_reply.started":"2021-06-22T14:26:40.886678Z","shell.execute_reply":"2021-06-22T14:26:40.900330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def select_sentence(text):\n    text_list = text.split('\\n')\n    text_set = {x for sentence in text_list for x in sentence.split('.')}\n    return {\n        sentence for sentence in text_set \n        if len(sentence.split()) >= 6\n    }","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.902844Z","iopub.execute_input":"2021-06-22T14:26:40.903337Z","iopub.status.idle":"2021-06-22T14:26:40.917019Z","shell.execute_reply.started":"2021-06-22T14:26:40.903300Z","shell.execute_reply":"2021-06-22T14:26:40.914006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_prediction(x, y, tokenizer, label2id):\n    decoded_predictions = set()\n    for input_ids, predictions in zip(x, y):\n        words = []\n        for i, prediction in enumerate(predictions[:len(input_ids)-1]):\n            if prediction == label2id['B-DATA']:\n                if words:\n                    decoded_predictions.add(tokenizer.decode(words))\n                    words.clear()\n                words.append(input_ids[i])\n            elif words:\n                if prediction == label2id['I-DATA']:\n                    words.append(input_ids[i])\n                else:\n                    decoded_predictions.add(tokenizer.decode(words))\n                    words.clear()\n        if words:\n            decoded_predictions.add(tokenizer.decode(words))\n    return decoded_predictions","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.918578Z","iopub.execute_input":"2021-06-22T14:26:40.919115Z","iopub.status.idle":"2021-06-22T14:26:40.927432Z","shell.execute_reply.started":"2021-06-22T14:26:40.919077Z","shell.execute_reply":"2021-06-22T14:26:40.926508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1st Stage: Text Matching with Additional Labels","metadata":{}},{"cell_type":"code","source":"test_ids = []\n\nprepared_data = {}\nfirst_stage_predictions = {}\ncleaned_addtional_labels = [clean_text(addtional_label) for addtional_label in addtional_labels]\nfor row in sample_df.itertuples():\n    \n    sample_text = row.text\n    test_id = row.Id\n    \n    cleaned_labels = set()\n    \n    sample_text_lower = f'{sample_text} '.lower()\n    for known_label in existing_labels:\n        if known_label in sample_text_lower:\n            cleaned_labels.add(clean_text(known_label))\n            \n    for addtional_label, cleaned_addtional_label in zip(addtional_labels, cleaned_addtional_labels):\n        if addtional_label in sample_text_lower:\n            if all(cleaned_addtional_label not in label for label in cleaned_labels):\n                cleaned_labels.add(cleaned_addtional_label)           \n    first_stage_predictions[test_id] = set(cleaned_labels)\n\n    test_ids.append(test_id)\n    \n    # preparing data for 2nd stage prediction\n    encoded_sentences = tokenizer(\n        list(select_sentence(sample_text)),\n        return_token_type_ids=False,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n    prepared_data[test_id] = {\n        'input_ids': pad_sequences(encoded_sentences['input_ids'], maxlen=MAX_LENGTH, padding='post'),\n        'attention_mask': pad_sequences(encoded_sentences['attention_mask'], maxlen=MAX_LENGTH, padding='post'),\n        'no_padded_input_ids': encoded_sentences['input_ids']\n    }","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:26:40.928690Z","iopub.execute_input":"2021-06-22T14:26:40.929273Z","iopub.status.idle":"2021-06-22T14:26:41.200084Z","shell.execute_reply.started":"2021-06-22T14:26:40.929236Z","shell.execute_reply":"2021-06-22T14:26:41.199201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2nd Stage: Named Entity Recognition","metadata":{}},{"cell_type":"code","source":"N_FOLDS = 4\nCNT_THRES = 2\neach_fold_predictions = {}\nsecond_stage_predictions = {}\nfor fold in range(N_FOLDS):\n    model_path = f'../input/coleridge-electra-base-ner4/fold{fold}/electra_base_crf'\n    transformer_layer = TFElectraForPreTraining.from_pretrained(ENCODER_DIR, output_hidden_states=True)\n    base_model = build_base_model(transformer_layer, num_cls=len(label2id), max_len=MAX_LENGTH)\n    model = ModelWithCRFLoss(base_model)\n    model.load_weights(model_path)\n    for test_id in test_ids:\n        x_test = prepared_data[test_id]\n        y_pred = model.predict(\n            {'input_ids': x_test['input_ids'], 'attention_mask': x_test['attention_mask']},\n            batch_size=BATCH_SIZE)[0]\n        labels = decode_prediction(\n            x_test['no_padded_input_ids'], \n            y_pred, \n            tokenizer, \n            label2id\n        )\n        \n        if test_id not in each_fold_predictions:\n            each_fold_predictions[test_id] = defaultdict(int)\n        \n        for label in labels:\n            each_fold_predictions[test_id][label] += 1\n\nsecond_stage_predictions = {\n    test_id: {\n        clean_text(label) for label, cnt in each_fold_predictions[test_id].items() if cnt >= CNT_THRES\n    } for test_id in test_ids\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:30:59.208937Z","iopub.execute_input":"2021-06-22T14:30:59.209303Z","iopub.status.idle":"2021-06-22T14:32:03.158480Z","shell.execute_reply.started":"2021-06-22T14:30:59.209271Z","shell.execute_reply":"2021-06-22T14:32:03.157594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post Processing","metadata":{}},{"cell_type":"code","source":"prediction_string_list = []\nfor test_id in test_ids:\n    first = first_stage_predictions[test_id]\n    second = set()\n    for ner_label in second_stage_predictions[test_id]:\n        cleaned_ner_label = clean_text(ner_label)\n        if all(\n            jaccard_similarity(cleaned_ner_label, cleaned_matching_label) < 0.5 \n            for cleaned_matching_label in first\n        ):\n            second.add(cleaned_ner_label)\n    prediction_string_list.append('|'.join(first | second ))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:32:03.159948Z","iopub.execute_input":"2021-06-22T14:32:03.160317Z","iopub.status.idle":"2021-06-22T14:32:03.170641Z","shell.execute_reply.started":"2021-06-22T14:32:03.160280Z","shell.execute_reply":"2021-06-22T14:32:03.169702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = test_ids\nsubmission['PredictionString'] = prediction_string_list\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T14:32:03.172647Z","iopub.execute_input":"2021-06-22T14:32:03.173079Z","iopub.status.idle":"2021-06-22T14:32:03.191610Z","shell.execute_reply.started":"2021-06-22T14:32:03.173044Z","shell.execute_reply":"2021-06-22T14:32:03.190683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}