{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING CONFIGS**\n\n* Training max sequence length: 320\n* Inference max sequence length: 320\n* Preprocessing win_size: 200 (words)\n* Batch size: 4\n* Head: Arcface (0.5, 10.0, easy_margin=True, centers=1)\n* Training SupportSet K: 3\n* Cased\n* Balanced Training Group Sampling\n* None Overlap Support Group Sampling\n* Support/Query groups are all unique groups.","metadata":{}},{"cell_type":"code","source":"WIN_SIZE = 200\nSEQUENCE_LENGTH = 320","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:36.637371Z","iopub.execute_input":"2021-06-24T12:36:36.637754Z","iopub.status.idle":"2021-06-24T12:36:36.643287Z","shell.execute_reply.started":"2021-06-24T12:36:36.637676Z","shell.execute_reply":"2021-06-24T12:36:36.642378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/pretrainedrobertabase'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:38.413583Z","iopub.execute_input":"2021-06-24T12:36:38.413909Z","iopub.status.idle":"2021-06-24T12:36:38.429626Z","shell.execute_reply.started":"2021-06-24T12:36:38.413858Z","shell.execute_reply":"2021-06-24T12:36:38.428911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nphysical_devices = tf.config.list_physical_devices(\"GPU\")\nfor i in range(len(physical_devices)):\n    tf.config.experimental.set_memory_growth(physical_devices[i], True)\n\nimport pandas as pd\nimport gc\nimport json\nimport numpy as np\nimport random\nfrom tqdm import tqdm\nimport transformers\nfrom transformers import *\nimport re\nfrom collections import Counter\nimport glob\nfrom functools import partial\nfrom multiprocessing import Pool\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ntf.random.set_seed(42)\nrandom.seed(42)\nnp.random.seed(42)\n\ntransformers.__version__","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:38.538275Z","iopub.execute_input":"2021-06-24T12:36:38.538539Z","iopub.status.idle":"2021-06-24T12:36:47.100772Z","shell.execute_reply.started":"2021-06-24T12:36:38.538514Z","shell.execute_reply":"2021-06-24T12:36:47.099973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_s_e_window_sliding(sample_len, win_size, step_size):\n    start = 0\n    end = win_size\n    s_e = []\n    s_e.append([start, end])\n    while end < sample_len:\n        start += step_size\n        end = start + win_size\n        s_e.append([start, end])\n\n    s_e[-1][0] -= s_e[-1][1] - sample_len\n    s_e[-1][0] = max(s_e[-1][0], 0)\n    s_e[-1][1] = sample_len\n    return s_e","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.103561Z","iopub.execute_input":"2021-06-24T12:36:47.103908Z","iopub.status.idle":"2021-06-24T12:36:47.109883Z","shell.execute_reply.started":"2021-06-24T12:36:47.103868Z","shell.execute_reply":"2021-06-24T12:36:47.109028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.111334Z","iopub.execute_input":"2021-06-24T12:36:47.111901Z","iopub.status.idle":"2021-06-24T12:36:47.234484Z","shell.execute_reply.started":"2021-06-24T12:36:47.111850Z","shell.execute_reply":"2021-06-24T12:36:47.233742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_label = train_df.cleaned_label.tolist()\ndataset_label = train_df.cleaned_label.tolist()\ndataset_title = train_df.dataset_title.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.235956Z","iopub.execute_input":"2021-06-24T12:36:47.236291Z","iopub.status.idle":"2021-06-24T12:36:47.248175Z","shell.execute_reply.started":"2021-06-24T12:36:47.236254Z","shell.execute_reply":"2021-06-24T12:36:47.244407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower().strip() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower().strip() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower().strip() for x in train_df['cleaned_label'].unique()]\nall_train_labels = list(set(temp_1 + temp_2 + temp_3))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.250827Z","iopub.execute_input":"2021-06-24T12:36:47.251270Z","iopub.status.idle":"2021-06-24T12:36:47.265557Z","shell.execute_reply.started":"2021-06-24T12:36:47.251232Z","shell.execute_reply":"2021-06-24T12:36:47.264474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IDS = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/**\")\nTEST_IDS = [TEST_ID.split(\"/\")[-1].split(\".\")[0] for TEST_ID in TEST_IDS]","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.268718Z","iopub.execute_input":"2021-06-24T12:36:47.268996Z","iopub.status.idle":"2021-06-24T12:36:47.280286Z","shell.execute_reply.started":"2021-06-24T12:36:47.268973Z","shell.execute_reply":"2021-06-24T12:36:47.279204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"win_size = WIN_SIZE\n\ndef process(i):\n    ids = []\n    texts = []\n    labels = []\n    pub_titles = []\n    cleaned_labels = []\n    x = json.load(open(\n        f\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/{TEST_IDS[i]}.json\",\"rt\"))\n    label = \"unknow\"\n    full_text = \"\"\n    unique_id = []\n    for section in x:\n        raw_text = section[\"text\"].replace(\"\\n\", \" \")\n#         raw_text_encode = tokenizer.encode(raw_text)[1:-1]\n        raw_text_encode = raw_text.split()\n        s_e = generate_s_e_window_sliding(len(raw_text_encode), win_size, int(0.75*win_size))\n        for (s, e) in s_e:\n#             sent = tokenizer.decode(raw_text_encode[s:e]).strip()\n            sent = \" \".join(raw_text_encode[s:e]).strip()\n            texts.append(sent)\n            ids.append(TEST_IDS[i])\n            labels.append(label)\n        full_text += section[\"text\"].replace(\"\\n\", \" \") + \" \"\n    \n    unique_id = TEST_IDS[i]\n    full_text = full_text.strip()\n\n    results = {}\n    results[\"id\"] = ids\n    results[\"text\"] = texts\n    results[\"label\"] = labels\n    results[\"unique_id\"] = unique_id\n    results[\"full_text\"] = full_text\n    return results\n        \n# define map iterator\ndef iterator_data(items_list):\n    for item in items_list:\n        yield item\n\niterator_data = iterator_data(range(len(TEST_IDS)))\np = Pool(8)\n\npartial_fn = partial(process)\ntrain_map = p.imap(\n    partial_fn,\n    tqdm(iterator_data, total=len(TEST_IDS), desc=\"[Preprocessing TestSet]\"),\n    chunksize=10,\n)\n\nresults = []\nfor result in tqdm(train_map):\n    results.append(result)\n\nids = []\ntexts = []\nlabels = []\nunique_ids = []\nfull_texts = []\nfor result in tqdm(results):\n    ids.extend(result[\"id\"])\n    texts.extend(result[\"text\"])\n    labels.extend(result[\"label\"])\n    unique_ids.append(result[\"unique_id\"])\n    full_texts.append(result[\"full_text\"])\n    \ntest_df = pd.DataFrame()\ntest_df[\"id\"] = ids\ntest_df[\"text\"] = texts\ntest_df[\"label\"] = labels\ntest_df[\"group\"] = [-1] * len(ids)\ntest_df[\"title\"] = [\"\"] * len(ids)\n\np.close()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:47.283123Z","iopub.execute_input":"2021-06-24T12:36:47.283355Z","iopub.status.idle":"2021-06-24T12:36:47.421933Z","shell.execute_reply.started":"2021-06-24T12:36:47.283332Z","shell.execute_reply":"2021-06-24T12:36:47.420842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Loader**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom tensorflow.keras.utils import Sequence\nimport numpy as np\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm\nfrom transformers import RobertaTokenizerFast\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport math\n\n\nclass QueryDataLoader(Sequence):\n    def __init__(self, data, batch_size=32):\n        self.batch_size = batch_size\n        self.data = data.fillna(\"\")\n        self.batch_ids = self.data[\"id\"].tolist()\n        self.batch_text = self.data[\"text\"].tolist()\n        self.batch_label = self.data[\"label\"].tolist()\n\n    def __len__(self):\n        return math.ceil(len(self.batch_text) / self.batch_size)\n\n    def __getitem__(self, index):\n        id = self.batch_ids[index * self.batch_size : (index + 1) * self.batch_size]\n        text = self.batch_text[index * self.batch_size : (index + 1) * self.batch_size]\n        label = self.batch_label[\n            index * self.batch_size : (index + 1) * self.batch_size\n        ]\n        classes = [1 if l != \"\" else 0 for l in label]\n        return id, text, label, classes\n\n\nclass SupportQueryDataLoader(Sequence):\n    def __init__(\n        self,\n        data,\n        tokenizer,\n        training_steps=500,\n        batch_size=32,\n        is_train=False,\n        query_dataloader=None,\n        query_masked=False,\n        return_query_ids=False,\n        return_query_labels=False,\n    ):\n        self.batch_size = batch_size\n        self.tokenizer = tokenizer\n        self.data = data.fillna(\"\")\n        self.is_train = is_train\n        self.len = training_steps\n        self.query_dataloader = query_dataloader\n        self.query_masked = query_masked\n        self.return_query_ids = return_query_ids\n        self.return_query_labels = return_query_labels\n\n        self.on_epoch_end()\n\n    def _create_group_data(self):\n        all_unique_group = list(self.data.group.unique())\n        for group in all_unique_group:\n            self.data_group[group] = list(\n                zip(\n                    list(self.data[self.data[\"group\"] == group].title),\n                    list(self.data[self.data[\"group\"] == group].text),\n                    list(self.data[self.data[\"group\"] == group].label),\n                )\n            )\n\n        self.all_unique_group = all_unique_group\n\n    def on_epoch_end(self):\n        if self.is_train:\n            for k in list(self.data_group.keys()):\n                self.data_group[k] = shuffle(self.data_group[k])\n\n    def __len__(self):\n        return self.len\n\n    def __getitem__(self, index):\n        # step 1: create support/group data samples\n        support_samples = []\n        support_labels = []\n        support_classes = []\n        query_samples = []\n        query_labels = []\n        query_classes = []\n        (\n            query_ids,\n            query_samples,\n            query_labels,\n            query_classes,\n        ) = self.query_dataloader.__getitem__(index)\n        if self.return_query_ids is False:\n            query_ids = None\n\n        # step 3: tokenize and return compute sequence label\n        query_batch = {}\n        query_batch[\"input_ids\"] = []\n        query_batch[\"attention_mask\"] = []\n        query_batch[\"token_type_ids\"] = []\n        query_batch[\"ids\"] = []\n\n        for i in range(len(query_samples)):\n            out = self._process_data(\n                query_samples[i], query_labels[i], self.query_masked\n            )\n            query_batch[\"input_ids\"].append(out[\"input_ids\"])\n            query_batch[\"attention_mask\"].append(out[\"attention_mask\"])\n            query_batch[\"token_type_ids\"].append(out[\"token_type_ids\"])\n            if query_ids is not None:\n                query_batch[\"ids\"].append(query_ids[i])\n\n        # step 4: padding to max len\n        query_batch[\"input_ids\"] = pad_sequences(\n            query_batch[\"input_ids\"],\n            padding=\"post\",\n            value=self.tokenizer.pad_token_id,\n        )\n        for k in [\"attention_mask\", \"token_type_ids\"]:\n            pad_value = 0\n            query_batch[k] = pad_sequences(\n                query_batch[k], padding=\"post\", value=pad_value\n            )\n        \n        for k in list([\"input_ids\", \"attention_mask\", \"token_type_ids\"]):\n            query_batch[k] = np.array(query_batch[k]).astype(np.int32)\n\n        return query_batch\n\n    def _process_data(self, inp_string, label_string, masked_label=False):\n        input_tokenize = self.tokenizer(\n            inp_string, return_offsets_mapping=True, max_length=SEQUENCE_LENGTH, truncation=True\n        )\n        results = {\n            \"input_ids\": input_tokenize[\"input_ids\"],\n            \"attention_mask\": input_tokenize[\"attention_mask\"],\n            \"token_type_ids\": [0] * len(input_tokenize[\"input_ids\"]),\n        }\n        return results","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:55.360166Z","iopub.execute_input":"2021-06-24T12:36:55.360523Z","iopub.status.idle":"2021-06-24T12:36:55.385011Z","shell.execute_reply.started":"2021-06-24T12:36:55.360490Z","shell.execute_reply":"2021-06-24T12:36:55.383974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MODELING**","metadata":{}},{"cell_type":"code","source":"def del_everything(model):\n    tf.keras.backend.clear_session()\n    del model\n    gc.collect()\n    sess = tf.compat.v1.keras.backend.get_session()\n    del sess\n    graph = tf.compat.v1.get_default_graph()\n    del graph","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:55.605721Z","iopub.execute_input":"2021-06-24T12:36:55.606041Z","iopub.status.idle":"2021-06-24T12:36:55.610633Z","shell.execute_reply.started":"2021-06-24T12:36:55.606012Z","shell.execute_reply":"2021-06-24T12:36:55.609586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom transformers.optimization_tf import WarmUp, AdamWeightDecay\nimport tensorflow_addons as tfa\nimport tensorflow.keras.backend as K\nimport math\n\n\nclass MetricLearningModel(tf.keras.Model):\n    def __init__(self, config, **kwargs):\n        super().__init__(**kwargs)\n        self.main_model = None\n        self.support_dense = tf.keras.layers.Dense(units=768, activation=None)\n        self.config = config\n        self.K = 3\n\n    def _compute_avg_embeddings(self, sequence_embeddings, attentions_mask, K=3):\n        embeddings = tf.reduce_mean(\n            attentions_mask * sequence_embeddings, axis=1\n        )  # [B * K, F]\n        if K > 1:\n            embeddings = tf.reshape(\n                embeddings,\n                (-1, K, self.support_dense.units),\n            )\n            embeddings = tf.reduce_mean(embeddings, axis=1)  # [B, F]\n        return embeddings\n\n    def call(\n        self,\n        inputs,\n        training=False,\n        sequence_labels=None,\n        mask_embeddings=None,\n        nomask_embeddings=None,\n        use_only_mask=False\n    ):\n        output_hidden_states = self.main_model(input_ids=inputs[0], attention_mask=inputs[1], training=training)[-2]\n        concat_hidden_states = tf.concat(\n            output_hidden_states[-1:], axis=-1\n        )  # [B * K, T, F]\n        concat_hidden_states = self.support_dense(\n            concat_hidden_states\n        )  # [B * K, T, 768]\n        sequence_embeddings = concat_hidden_states[:, 0, :]  # [B * K, 768]\n        if sequence_labels is not None:\n            sequence_labels = tf.cast(\n                sequence_labels, dtype=concat_hidden_states.dtype\n            )[..., None]\n            mask_embeddings = self._compute_avg_embeddings(\n                concat_hidden_states,\n                tf.where(sequence_labels == -100, 0.0, sequence_labels),\n                self.K,\n            )\n            nomask_embeddings = self._compute_avg_embeddings(\n                concat_hidden_states,\n                1.0 - tf.where(sequence_labels == -100, 1.0, sequence_labels),\n                K=self.K,\n            )\n            return sequence_embeddings, mask_embeddings, nomask_embeddings\n        else:\n            attention_mask = tf.cast(inputs[1], concat_hidden_states.dtype)[\n                ..., None\n            ]  # [B, T, 1]\n            normed_mask_embeddings = tf.nn.l2_normalize(mask_embeddings, axis=1)[..., None]\n            normed_nomask_embeddings = tf.nn.l2_normalize(nomask_embeddings, axis=1)[..., None]\n            normed_hidden_states = tf.nn.l2_normalize(concat_hidden_states, axis=-1)\n            mask_cosine_similarity = tf.matmul(\n                normed_hidden_states, normed_mask_embeddings\n            )  # [B, T, 1]\n            nomask_cosine_similarity = tf.matmul(\n                normed_hidden_states, normed_nomask_embeddings\n            )  # [B, T, 1]\n            mask_attentions = tf.nn.sigmoid(10.0 * mask_cosine_similarity)  # [B, T, 1]\n            nomask_attentions = tf.nn.sigmoid(10.0 * nomask_cosine_similarity)  # [B, T, 1]\n\n            # average attention\n            if use_only_mask:\n                attentions = mask_attentions\n            else:\n                attentions = 0.5 * (mask_attentions + (1.0 - nomask_attentions))\n\n            attentions *= attention_mask\n\n            # compute mask and nomask embeddings\n            mask_embeddings = self._compute_avg_embeddings(\n                concat_hidden_states,\n                tf.where(attention_mask == 0, 0.0, attentions),\n                K=1,\n            )\n            nomask_embeddings = self._compute_avg_embeddings(\n                concat_hidden_states,\n                1.0 - tf.where(attention_mask == 0, 1.0, attentions),\n                K=1,\n            )\n            return sequence_embeddings, mask_embeddings, nomask_embeddings, attentions","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:55.758277Z","iopub.execute_input":"2021-06-24T12:36:55.758564Z","iopub.status.idle":"2021-06-24T12:36:55.891056Z","shell.execute_reply.started":"2021-06-24T12:36:55.758539Z","shell.execute_reply":"2021-06-24T12:36:55.890207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference**","metadata":{}},{"cell_type":"code","source":"def find_all_start_end(attention_values):\n    start_offset = {}\n    current_idx = 0\n    is_start = False\n    start_end = []\n    while current_idx < len(attention_values):\n        if attention_values[current_idx] == 1 and is_start is False:\n            start_offset[current_idx] = 0\n            is_start = True\n            start_idx = current_idx\n        elif attention_values[current_idx] == 1 and is_start is True:\n            start_offset[start_idx] += 1\n        elif attention_values[current_idx] == 0 and is_start is True:\n            is_start = False\n        current_idx += 1\n    for k, v in start_offset.items():\n        start_end.append([k, k + v + 1])\n    return start_end","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:56.033382Z","iopub.execute_input":"2021-06-24T12:36:56.033622Z","iopub.status.idle":"2021-06-24T12:36:56.040154Z","shell.execute_reply.started":"2021-06-24T12:36:56.033598Z","shell.execute_reply":"2021-06-24T12:36:56.039270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_cosine_similarity(x1, x2):\n    x1_norm = tf.nn.l2_normalize(x1, axis=1)\n    x2_norm = tf.nn.l2_normalize(x2, axis=1)\n    cosine_similarity = tf.matmul(x1_norm, x2_norm, transpose_b=True)  # [B1, B2]\n    return tf.clip_by_value(cosine_similarity, -1.0, 1.0)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:56.174528Z","iopub.execute_input":"2021-06-24T12:36:56.174770Z","iopub.status.idle":"2021-06-24T12:36:56.181576Z","shell.execute_reply.started":"2021-06-24T12:36:56.174747Z","shell.execute_reply":"2021-06-24T12:36:56.180804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference(test_dataloader, \n                  model, all_support_embeddings, \n                  all_support_mask_embeddings, \n                  all_support_nomask_embeddings, ner_threshold=[0.5, 0.7]):\n    preds = []\n    preds_low_confidence = []\n    cosines = []\n    ids = []\n    text_ids = []\n    inputs = []\n    N_TTA = 100\n    \n    tokenizer = test_dataloader.tokenizer\n\n    for query_batch in tqdm(test_dataloader):\n        all_cosines = []\n        support_embeddings = all_support_embeddings[\n            np.random.choice(range(all_support_embeddings.shape[0]), \n                             size=query_batch[\"input_ids\"].shape[0] * N_TTA)\n        ]\n        support_mask_embeddings = all_support_mask_embeddings[\n            np.random.choice(range(all_support_mask_embeddings.shape[0]), \n                             size=query_batch[\"input_ids\"].shape[0] * N_TTA)\n        ]\n        support_nomask_embeddings = all_support_nomask_embeddings[\n            np.random.choice(range(all_support_nomask_embeddings.shape[0]), \n                             size=query_batch[\"input_ids\"].shape[0] * N_TTA)\n        ]\n        support_mask_embeddings = np.mean(np.reshape(support_mask_embeddings, (-1, N_TTA, 768)), axis=1)\n        support_nomask_embeddings = np.mean(np.reshape(support_nomask_embeddings, (-1, N_TTA, 768)), axis=1)\n        query_embeddings, query_mask_embeddings, query_nomask_embeddings, attention_values = model(\n            [\n                query_batch[\"input_ids\"],\n                query_batch[\"attention_mask\"],\n            ],\n            training=False,\n            sequence_labels=None,\n            mask_embeddings=support_mask_embeddings,\n            nomask_embeddings=support_nomask_embeddings,\n        )  # [B, F]\n        cosine = compute_cosine_similarity(query_embeddings, support_embeddings).numpy()\n        cosine = np.mean(cosine, axis=1)\n        all_cosines.extend(cosine)\n        ids.extend(query_batch[\"ids\"])\n        for k in range(len(all_cosines)):\n            for TH in ner_threshold:\n                binary_pred_at_th = attention_values.numpy()[k, :, 0] >= TH\n                if np.sum(binary_pred_at_th) > 0:\n                    binary_pred_at_th = binary_pred_at_th.astype(np.int32)\n                    start_end = find_all_start_end(binary_pred_at_th)\n                    pred_candidates = []\n                    for s_e in start_end:\n                        if (s_e[1] - s_e[0]) >= 4:\n                            pred_tokens = list(range(s_e[0], s_e[1]))\n                            pred = tokenizer.decode(query_batch[\"input_ids\"][k, ...][pred_tokens])\n                            pred_candidates.append(pred)\n                    pred = \"|\".join(pred_candidates)\n                else:\n                    pred = \"\"\n                if TH == 0.7:\n                    preds.append(pred)\n                else:\n                    preds_low_confidence.append(pred)\n            cosines.append(all_cosines[k])\n    return ids, text_ids, inputs, cosines, preds, preds_low_confidence","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:56.349181Z","iopub.execute_input":"2021-06-24T12:36:56.349426Z","iopub.status.idle":"2021-06-24T12:36:56.363139Z","shell.execute_reply.started":"2021-06-24T12:36:56.349402Z","shell.execute_reply":"2021-06-24T12:36:56.362132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def end2end(pretrained_path, checkpoint_path, test_df, ner_threshold=[0.5, 0.7]):\n    config = AutoConfig.from_pretrained(\n        f\"/kaggle/input/{pretrained_path}/\")\n    config.output_attentions = True\n    config.output_hidden_states = True\n\n    main_model = TFAutoModel.from_config(config=config)\n    model = MetricLearningModel(config=config, name=\"metric_learning_model\")\n    model.main_model = main_model\n    model.K = 3\n    \n    # load pre-extract embedding\n    checkpoint_path = f\"/kaggle/input/{checkpoint_path}\"\n    all_support_embeddings = np.load(os.path.join(checkpoint_path, \"support_embeddings.npy\"))\n    all_support_mask_embeddings = np.load(os.path.join(checkpoint_path, \"support_mask_embeddings.npy\"))\n    all_support_nomask_embeddings = np.load(os.path.join(checkpoint_path, \"support_nomask_embeddings.npy\"))\n    \n    \n    # create tokenizer and dataloader\n    if \"distil\" in pretrained_path:\n        tokenizer = DistilBertTokenizerFast.from_pretrained(f\"/kaggle/input/{pretrained_path}/\")\n    elif \"roberta\" in pretrained_path:\n        tokenizer = RobertaTokenizerFast.from_pretrained(f\"/kaggle/input/{pretrained_path}/\")\n    elif \"scibert\" in pretrained_path:\n        tokenizer = BertTokenizerFast.from_pretrained(f\"/kaggle/input/{pretrained_path}/\", do_lower_case=False)\n\n    query_dataloader = QueryDataLoader(test_df, batch_size=128)\n    test_dataloader = SupportQueryDataLoader(\n        test_df,\n        tokenizer=tokenizer,\n        batch_size=128,\n        is_train=False,\n        training_steps=len(query_dataloader),\n        query_dataloader=query_dataloader,\n        return_query_ids=True,\n    )\n    \n    # build model with real input and load_weights\n    query_batch = test_dataloader.__getitem__(0)\n    (\n        query_embeddings,\n        query_mask_embeddings,\n        query_nomask_embeddings,\n        attention_values,\n    ) = model(\n        [\n            query_batch[\"input_ids\"][:1, ...],\n            query_batch[\"attention_mask\"][:1, ...],\n        ],\n        training=True,\n        sequence_labels=None,\n        mask_embeddings=all_support_mask_embeddings[:1, ...],\n        nomask_embeddings=all_support_nomask_embeddings[:1, ...],\n    )  # [B, F]\n    model.summary()\n    weights_path = glob.glob(os.path.join(checkpoint_path, \"*.h5\"))[0]\n    model.load_weights(weights_path, by_name=True)\n    \n    # apply tf.function\n    model = tf.function(model, experimental_relax_shapes=True)\n    \n    # run inference\n    ids, text_ids, inputs, cosines, preds, preds_low_confidence = run_inference(\n        test_dataloader, \n        model, \n        all_support_embeddings, \n        all_support_mask_embeddings, \n        all_support_nomask_embeddings,\n        ner_threshold=ner_threshold\n    )\n    \n    # release model\n    del_everything(model)\n    \n    return ids, text_ids, inputs, cosines, preds, preds_low_confidence, test_dataloader.tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:56.502182Z","iopub.execute_input":"2021-06-24T12:36:56.502418Z","iopub.status.idle":"2021-06-24T12:36:56.514166Z","shell.execute_reply.started":"2021-06-24T12:36:56.502396Z","shell.execute_reply":"2021-06-24T12:36:56.513128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nstop_words = set(stopwords.words('english')) \n\ndef remove_stopwords(string):\n    word_tokens = word_tokenize(string) \n    filtered_sentence = [w for w in word_tokens if not w in stop_words]\n    return \" \".join(filtered_sentence).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:56.621827Z","iopub.execute_input":"2021-06-24T12:36:56.622112Z","iopub.status.idle":"2021-06-24T12:36:57.120917Z","shell.execute_reply.started":"2021-06-24T12:36:56.622088Z","shell.execute_reply":"2021-06-24T12:36:57.120062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_special_token(string, tokenizer):\n    pad_token = tokenizer.pad_token\n    sep_token = tokenizer.sep_token\n    cls_token = tokenizer.cls_token\n    \n    if (pad_token not in string) and (sep_token not in string) and (cls_token not in string):\n        return True\n    return False","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:57.122662Z","iopub.execute_input":"2021-06-24T12:36:57.122990Z","iopub.status.idle":"2021-06-24T12:36:57.130549Z","shell.execute_reply.started":"2021-06-24T12:36:57.122962Z","shell.execute_reply":"2021-06-24T12:36:57.129655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt, lower=True):\n    if lower:\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n    else:\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:57.132232Z","iopub.execute_input":"2021-06-24T12:36:57.132598Z","iopub.status.idle":"2021-06-24T12:36:57.138641Z","shell.execute_reply.started":"2021-06-24T12:36:57.132560Z","shell.execute_reply":"2021-06-24T12:36:57.137629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(str1, str2): \n    a = set(str1.lower().split(\" \"))\n    b = set(str2.lower().split(\" \"))\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:57.140186Z","iopub.execute_input":"2021-06-24T12:36:57.140538Z","iopub.status.idle":"2021-06-24T12:36:57.149468Z","shell.execute_reply.started":"2021-06-24T12:36:57.140487Z","shell.execute_reply":"2021-06-24T12:36:57.148826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_cased_pred(lower_start_idx, lower_end_idx, lower_string, cased_string, lower_pred):\n    len_lower_string = len(lower_string)\n    len_cased_string = len(cased_string)\n    if (len_lower_string - len_cased_string) == 0:\n        return cased_string[lower_start_idx: lower_end_idx]\n    else:\n        diff_len = abs(len_lower_string - lower_end_idx)\n        for shift_idx in range(-diff_len - 1, diff_len + 1):\n            cased_pred_candidate = cased_string[lower_start_idx + shift_idx : lower_start_idx + shift_idx + len(lower_pred)]\n            if cased_pred_candidate.lower() == lower_pred:\n                return cased_pred_candidate\n    return lower_pred.upper()\n\n\ndef calculate_iou(se_0, se_1):\n    s_0, e_0 = se_0\n    s_1, e_1 = se_1    \n    max_s = max(s_0, s_1)\n    min_e = min(e_0, e_1)\n    intersection = (min_e - max_s)\n    return  intersection / ((e_0 - s_0) + (e_1 - s_1) - intersection)\n\n\ndef find_all_pred_in_text(normed_text_cased, all_unique_preds):\n    normed_text_cased = clean_text(normed_text_cased, False)\n    normed_text = normed_text_cased.lower()\n    preds = []\n    preds_indexs = []\n    for pred in all_unique_preds:\n        if (\" \" + pred + \" \" in normed_text) or (\" \" + pred + \",\" in normed_text) or (\" \" + pred + \".\" in normed_text):\n            preds.append(pred)\n    unique_preds = [] # unique in terms of index. \n    preds = list(sorted(preds, key=len))\n    for pred in preds:\n        matchs = re.finditer(pred, normed_text)\n        for match in matchs:\n            start_index = match.start()\n            end_index = match.end()\n            pred_cased = find_cased_pred(start_index, end_index, normed_text, normed_text_cased, pred)\n            if pred_cased.islower() is False:\n                preds_indexs.append([start_index, end_index])\n                unique_preds.append(pred)\n    group_idxs = []\n    for i in range(len(preds_indexs)):\n        for j in range(len(preds_indexs)):\n            if i != j:\n                start_i, end_i = preds_indexs[i]\n                start_j, end_j = preds_indexs[j]\n                iou = calculate_iou(preds_indexs[i], preds_indexs[j])\n                if (start_i <= end_j and end_i <= end_j and start_i >= start_j) or iou >= 0.1:\n                    group_idxs.append([i, j])\n    unique_preds = np.array(unique_preds)\n    for group_idx in group_idxs:\n        unique_preds[group_idx[0]] = unique_preds[group_idx[1]]\n    return np.unique(unique_preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:57.698821Z","iopub.execute_input":"2021-06-24T12:36:57.699137Z","iopub.status.idle":"2021-06-24T12:36:57.713525Z","shell.execute_reply.started":"2021-06-24T12:36:57.699109Z","shell.execute_reply":"2021-06-24T12:36:57.712693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove acronym from prediction\ndef remove_acronym(preds):\n    for i in range(len(preds)):\n        pred_i = preds[i]\n        pred_i = pred_i.replace(\"( \", \"(\")\n        pred_i = pred_i.replace(\" )\", \")\")\n        pred_i = pred_i.replace(\"[ \", \"[\")\n        pred_i = pred_i.replace(\" ]\", \"]\")\n        try:\n            new_pred_i = []\n            for pi in pred_i.split(\"|\"):\n                if pi != \"\":\n                    words = pi.split()\n                    if \"(\" in words[-1] or \"[\" in words[-1]:\n                        new_pred_i.append(\" \".join(words[:-1]))\n                    else:\n                        new_pred_i.append(pi)\n            new_pred_i = \"|\".join(new_pred_i)\n            preds[i] = new_pred_i\n        except:\n            pass\n    return preds","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:57.873560Z","iopub.execute_input":"2021-06-24T12:36:57.873829Z","iopub.status.idle":"2021-06-24T12:36:57.882393Z","shell.execute_reply.started":"2021-06-24T12:36:57.873806Z","shell.execute_reply":"2021-06-24T12:36:57.881577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_overlap(preds, preds_low_confidence):\n    for i in range(len(preds_low_confidence)):\n        if preds[i] == \"\" or preds_low_confidence[i] == \"\":\n            continue\n        pred_i = preds[i].split(\"|\")\n        pred_low_conf_i = preds_low_confidence[i].split(\"|\")\n        new_p_low = []\n        for p_low in pred_low_conf_i:\n            overlap = False\n            for p in pred_i:\n                if p in p_low:\n                    overlap = True\n                    break\n            if overlap is False:\n                new_p_low.append(p_low)\n        if len(new_p_low) == 0:\n            preds_low_confidence[i] = \"\"\n        else:\n            preds_low_confidence[i] = \"|\".join(new_p_low)\n    return preds_low_confidence","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:58.868321Z","iopub.execute_input":"2021-06-24T12:36:58.868623Z","iopub.status.idle":"2021-06-24T12:36:58.875373Z","shell.execute_reply.started":"2021-06-24T12:36:58.868595Z","shell.execute_reply":"2021-06-24T12:36:58.874276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_valid_low_confidence_pred(pred):\n    clean_pred = clean_text(pred, True)\n    keywords = [\"study\", \"survey\", \"studies\", \"database\", \"dataset\", \"data system\", \"system data\", \"data set\", \"data base\", \"program\"]\n    if pred != \"\":\n        words = pred.strip().split()\n        clean_words = clean_pred.strip().split()\n        string_check= re.compile('[\\(\\)\\[\\]]')\n        if clean_words[0] in [\"a\", \"an\", \"the\"]:\n            return False\n        if clean_words[-1] in [\"a\", \"an\", \"the\", \"in\", \"on\", \"of\", \"for\", \"and\", \"or\"]:\n            return False\n        if words[0][0].isalpha() and words[0][0].isupper() and string_check.search(words[0]) is None:\n            for kw in keywords:\n                if kw in clean_pred:\n                    return True\n    return False","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:36:59.014176Z","iopub.execute_input":"2021-06-24T12:36:59.014420Z","iopub.status.idle":"2021-06-24T12:36:59.021502Z","shell.execute_reply.started":"2021-06-24T12:36:59.014395Z","shell.execute_reply":"2021-06-24T12:36:59.020487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create text per id\nraw_text_per_id = {}\nclean_text_per_id = {}\nall_unique_ids = unique_ids\n\nfor i in tqdm(range(len(all_unique_ids)), desc=\"Create raw text per id\"):\n    full_text = full_texts[i]\n    if id not in raw_text_per_id:\n        raw_text_per_id[all_unique_ids[i]] = full_text\n        clean_text_per_id[all_unique_ids[i]] = clean_text(full_text).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:20.701176Z","iopub.execute_input":"2021-06-24T12:40:20.701544Z","iopub.status.idle":"2021-06-24T12:40:20.742924Z","shell.execute_reply.started":"2021-06-24T12:40:20.701509Z","shell.execute_reply":"2021-06-24T12:40:20.741991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get all accepted preds\ndef get_accepted_preds(preds, preds_low_confidence, cosines, cosine_threshold, tokenizer):\n    accepted_preds = []\n    ########################################################\n    all_accepted_preds = []\n    for i in range(len(preds)):\n        if cosines[i] >= cosine_threshold:\n            a = preds[i].split(\"|\")\n            unique_v = np.unique(a)\n            all_accepted_preds.extend(unique_v)\n        else:\n            preds_low_confidence_i = preds_low_confidence[i].split(\"|\")\n            preds_low_confidence_i.extend(preds[i].split(\"|\"))\n            preds_low_confidence[i] = \"|\".join(preds_low_confidence_i)\n            \n            \n    counter_all_accepted_preds = Counter(all_accepted_preds)\n    for k, v in counter_all_accepted_preds.items():\n        k = k.strip()\n        if (\"#\" not in k) and len(clean_text(k).strip().split(\" \")) >= 3 and len(k.split(\" \")) >= 3 and len(remove_stopwords(k).split(\" \")) >= 3 and len(k) >= 10 and check_special_token(k, tokenizer):\n            if v >= 4:\n                accepted_preds.append(clean_text(k).strip())\n            else:\n                if check_valid_low_confidence_pred(k):\n                    accepted_preds.append(clean_text(k).strip())\n\n    ########################################################\n    all_accepted_preds = []\n    for i in range(len(preds_low_confidence)):\n        if cosines[i] >= -1.0:\n            a = preds_low_confidence[i].split(\"|\")\n            unique_v = np.unique(a)\n            all_accepted_preds.extend(unique_v)\n    counter_all_accepted_preds = Counter(all_accepted_preds)\n    for k, v in counter_all_accepted_preds.items():\n        k = k.strip()\n        if (\"#\" not in k) and len(clean_text(k).strip().split(\" \")) >= 3 and len(k.split(\" \")) >= 3 and len(remove_stopwords(k).split(\" \")) >= 3 and len(k) >= 10 and check_special_token(k, tokenizer):\n            if check_valid_low_confidence_pred(k):\n                accepted_preds.append(clean_text(k).strip())\n\n    accepted_preds = list(set(accepted_preds))\n    return accepted_preds","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:22.087650Z","iopub.execute_input":"2021-06-24T12:40:22.088029Z","iopub.status.idle":"2021-06-24T12:40:22.101090Z","shell.execute_reply.started":"2021-06-24T12:40:22.087977Z","shell.execute_reply":"2021-06-24T12:40:22.100240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accepted_preds = []\n\n\nPARAMS = [\n    (\"pretrainedbiomedrobertabase\", \"coleridgeinitiativebiomedrobertabasev2\", [0.5, 0.7], -0.1),\n    (\"scibertbasecased\", \"coleridgeinitiativescibertbasecasedv10\", [0.5, 0.7], -0.7),\n]\n\nfor i, param in enumerate(PARAMS):\n    ids, text_ids, inputs, cosines, preds, preds_low_confidence, tokenizer = end2end(\n        param[0], \n        param[1], \n        test_df,\n        ner_threshold=param[2])\n\n    preds = remove_acronym(preds)\n    preds_low_confidence = remove_acronym(preds_low_confidence)\n    preds_low_confidence = remove_overlap(preds, preds_low_confidence)\n    accepted_preds.extend(get_accepted_preds(preds, preds_low_confidence, cosines, param[3], tokenizer))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:22.227685Z","iopub.execute_input":"2021-06-24T12:40:22.227942Z","iopub.status.idle":"2021-06-24T12:40:54.284705Z","shell.execute_reply.started":"2021-06-24T12:40:22.227917Z","shell.execute_reply":"2021-06-24T12:40:54.283778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accepted_preds = list(set(accepted_preds))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.286648Z","iopub.execute_input":"2021-06-24T12:40:54.287014Z","iopub.status.idle":"2021-06-24T12:40:54.291497Z","shell.execute_reply.started":"2021-06-24T12:40:54.286986Z","shell.execute_reply":"2021-06-24T12:40:54.290380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make Submission**","metadata":{}},{"cell_type":"code","source":"accepted_preds","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.292701Z","iopub.execute_input":"2021-06-24T12:40:54.293046Z","iopub.status.idle":"2021-06-24T12:40:54.303650Z","shell.execute_reply.started":"2021-06-24T12:40:54.293012Z","shell.execute_reply":"2021-06-24T12:40:54.302251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_label_per_id = {}\n\nfor i in tqdm(range(len(all_unique_ids))):\n    full_text = raw_text_per_id[all_unique_ids[i]]\n    merged_pred_labels = find_all_pred_in_text(full_text, np.unique(accepted_preds))\n    group_label_per_id[all_unique_ids[i]] = []\n    group_label_per_id[all_unique_ids[i]].extend(merged_pred_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.305057Z","iopub.execute_input":"2021-06-24T12:40:54.305405Z","iopub.status.idle":"2021-06-24T12:40:54.347277Z","shell.execute_reply.started":"2021-06-24T12:40:54.305354Z","shell.execute_reply":"2021-06-24T12:40:54.346345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_valid_ac(long_form, short_form):\n    long_form = \"\".join([w[0] for w in long_form.split()])\n    short_form_candidate1 = \"\".join(\n        [w if i == 0 else w[0] for i, w in enumerate(short_form.split())]\n    )\n    short_form_candidate2 = short_form.split()[0]\n    short_form_accepted = None\n    original_long_index = len(long_form) - 1\n    for i, short_form_candidate in enumerate([short_form_candidate1, short_form_candidate2]):\n        long_index = len(long_form) - 1\n        short_index = len(short_form_candidate) - 1\n\n        while short_index >= 0:\n            current_charactor = short_form_candidate[short_index]\n            if not current_charactor.isalpha():\n                short_index -= 1\n                continue\n\n            while long_form[long_index] != current_charactor:\n                long_index -= 1\n                if long_index < 0:\n                    break\n\n            short_index -= 1\n            if long_index < 0:\n                break\n                \n        if long_index >= 0 and (not short_form.isdigit()) and long_index < original_long_index:\n            if i == 0:\n                short_form_accepted = short_form\n            else:\n                short_form_accepted = short_form.split()[0]\n                \n            if not (short_form_accepted[-1].isalpha() or short_form_accepted[-1].isdigit()):\n                short_form_accepted = short_form_accepted[:-1]\n            return short_form_accepted\n\n    return short_form_accepted","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.349961Z","iopub.execute_input":"2021-06-24T12:40:54.350276Z","iopub.status.idle":"2021-06-24T12:40:54.359693Z","shell.execute_reply.started":"2021-06-24T12:40:54.350243Z","shell.execute_reply":"2021-06-24T12:40:54.358817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text_v2(txt):\n    return re.sub('[^A-Za-z0-9\\(\\)\\[\\]]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.361725Z","iopub.execute_input":"2021-06-24T12:40:54.362291Z","iopub.status.idle":"2021-06-24T12:40:54.372809Z","shell.execute_reply.started":"2021-06-24T12:40:54.362253Z","shell.execute_reply":"2021-06-24T12:40:54.372080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_all_acronyms_candidates(group_label_per_id):\n    for k in group_label_per_id.keys():\n        string = clean_text_v2(raw_text_per_id[k])\n        all_labels = group_label_per_id[k].split(\"|\")\n        for label in all_labels:\n            if label != \"\":\n                acronyms_candidates = re.findall(f\"{label} \\((.*?)\\)\", string)\n                acronyms_candidates.extend(re.findall(f\"{label} \\[(.*?)\\]\", string))\n                acronyms_candidates = np.unique([ac for ac in acronyms_candidates if len(ac.split()) >= 1])\n                if len(acronyms_candidates) > 0:\n                    for ac in acronyms_candidates:\n                        ac = find_valid_ac(label, ac)\n                        if ac is not None:\n                            if len(ac.split(\" \")) <= 2:\n                                group_label_per_id[k] += f\"|{ac}\"\n    return group_label_per_id","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.375594Z","iopub.execute_input":"2021-06-24T12:40:54.376096Z","iopub.status.idle":"2021-06-24T12:40:54.384702Z","shell.execute_reply.started":"2021-06-24T12:40:54.376059Z","shell.execute_reply":"2021-06-24T12:40:54.383510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, v in group_label_per_id.items():\n    unique_v = list(np.unique(v))\n    if len(unique_v) >= 2:\n        group_label_per_id[k] = \"|\".join([v for v in unique_v if v != ''])\n    elif len(unique_v) == 1 and unique_v[0] == '':\n        group_label_per_id[k] = ''\n    elif len(unique_v) == 1 and unique_v[0] != '':\n        group_label_per_id[k] = f'{unique_v[0]}'\n    else:\n        group_label_per_id[k] = ''","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.386410Z","iopub.execute_input":"2021-06-24T12:40:54.386860Z","iopub.status.idle":"2021-06-24T12:40:54.396621Z","shell.execute_reply.started":"2021-06-24T12:40:54.386819Z","shell.execute_reply":"2021-06-24T12:40:54.395868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_label_per_id = find_all_acronyms_candidates(group_label_per_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.398052Z","iopub.execute_input":"2021-06-24T12:40:54.398314Z","iopub.status.idle":"2021-06-24T12:40:54.432912Z","shell.execute_reply.started":"2021-06-24T12:40:54.398289Z","shell.execute_reply":"2021-06-24T12:40:54.432218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = []\ny_ids = []\nfor k in list(group_label_per_id.keys()):\n    pred = []\n    pred.extend(group_label_per_id[k].split(\"|\"))\n    pred = np.unique(pred)\n    accepted_pred = []\n    for i in range(len(pred)):\n        clean_pred = clean_text(pred[i])\n        pred[i] = clean_pred.strip()\n        accepted_pred.append(pred[i])\n    y_pred.append(list(pred))\n    y_ids.append(k)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.434229Z","iopub.execute_input":"2021-06-24T12:40:54.434734Z","iopub.status.idle":"2021-06-24T12:40:54.442593Z","shell.execute_reply.started":"2021-06-24T12:40:54.434668Z","shell.execute_reply":"2021-06-24T12:40:54.441490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_merged = []\nfor pred in y_pred:\n    pred = \"|\".join(pred).strip()\n    y_pred_merged.append(pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.444146Z","iopub.execute_input":"2021-06-24T12:40:54.444639Z","iopub.status.idle":"2021-06-24T12:40:54.451608Z","shell.execute_reply.started":"2021-06-24T12:40:54.444603Z","shell.execute_reply":"2021-06-24T12:40:54.450490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = y_ids\nsubmission['PredictionString'] = y_pred_merged\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.452812Z","iopub.execute_input":"2021-06-24T12:40:54.453186Z","iopub.status.idle":"2021-06-24T12:40:54.462917Z","shell.execute_reply.started":"2021-06-24T12:40:54.453149Z","shell.execute_reply":"2021-06-24T12:40:54.462228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-24T12:40:54.464297Z","iopub.execute_input":"2021-06-24T12:40:54.464699Z","iopub.status.idle":"2021-06-24T12:40:55.168702Z","shell.execute_reply.started":"2021-06-24T12:40:54.464663Z","shell.execute_reply":"2021-06-24T12:40:55.167870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}