{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🤗 Bert for Question Answering Baseline: Inference\n\nThis code is adapted from my work in the Tweet Sentiment Extraction Competition.\n\nIt tackles the task as a Question Answering one, where the question is implicit and can be understood as : \"Which datasets are mentionned ?\"\n\n\nThe approach is quite naïve and has a lot of flaws. Feel free to ask any question in the comments.\n\nTraining Kernel : https://www.kaggle.com/theoviel/bert-for-question-answering-baseline-training","metadata":{}},{"cell_type":"markdown","source":"# Initialization","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport glob\nimport json\nimport torch\nimport datetime\nimport tokenizers\nimport numpy as np\nimport transformers\nimport pandas as pd\nimport torch.nn as nn\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tokenizers import *\nfrom functools import partial\nfrom tqdm.notebook import tqdm\nfrom torch.nn import functional as F\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:03.17311Z","iopub.execute_input":"2021-06-02T01:46:03.173974Z","iopub.status.idle":"2021-06-02T01:46:05.992987Z","shell.execute_reply.started":"2021-06-02T01:46:03.17361Z","shell.execute_reply":"2021-06-02T01:46:05.99093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Params","metadata":{}},{"cell_type":"code","source":"HOW = 'QA_ONLY' # 'MATCH_ONLY', 'QA_ONLY', 'UNION_MERGE', 'LAM', 'ALL_BLENDED_PP', 'ORIGINAL'\nTHRESHOLD = 0.999\nLAM_FL_TH = 0.5\nCP_PATH = '../input/bert-for-question-answering-baseline-training' + '/'\n\nSEED = 2020\n\nDATA_PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\nDATA_PATH_TRAIN = DATA_PATH + 'train/'\nDATA_PATH_TEST = DATA_PATH + 'test/'\n\nNUM_WORKERS = 4\n\nVOCABS = {\n    \"bert-base-uncased\": \"../input/vocabs/bert-base-uncased-vocab.txt\",\n}\n\nMODEL_PATHS = {\n    'bert-base-uncased': '../input/bertconfigs/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/',\n    'bert-large-uncased-whole-word-masking-finetuned-squad': '../input/bertconfigs/wwm_uncased_L-24_H-1024_A-16/wwm_uncased_L-24_H-1024_A-16/',\n    'albert-large-v2': '../input/albert-configs/albert-large-v2/albert-large-v2/',\n    'albert-base-v2': '../input/albert-configs/albert-base-v2/albert-base-v2/',\n    'distilbert': '../input/albert-configs/distilbert/distilbert/',\n}","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:05.996355Z","iopub.execute_input":"2021-06-02T01:46:05.996735Z","iopub.status.idle":"2021-06-02T01:46:06.004783Z","shell.execute_reply.started":"2021-06-02T01:46:05.996697Z","shell.execute_reply":"2021-06-02T01:46:06.003279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    # General\n    k = 5\n    seed = 2021\n\n    # Texts\n    max_len = 256\n    \n    # Architecture\n    selected_model = \"bert-base-uncased\"\n    lowercase = True\n    \n    # Training\n    batch_size = 16\n    batch_size_val = batch_size * 2","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.006538Z","iopub.execute_input":"2021-06-02T01:46:06.006903Z","iopub.status.idle":"2021-06-02T01:46:06.022429Z","shell.execute_reply.started":"2021-06-02T01:46:06.00687Z","shell.execute_reply":"2021-06-02T01:46:06.021341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"markdown","source":"## Tokenizer","metadata":{}},{"cell_type":"code","source":"class EncodedText:\n    def __init__(self, ids, offsets):\n        self.ids = ids\n        self.offsets = offsets\n\n\ndef create_tokenizer_and_tokens(config):\n    if \"roberta\" in config.selected_model:\n        raise NotImplementedError\n        \n    elif \"albert\" in config.selected_model:\n        raise NotImplementedError\n        \n    else:\n        tokenizer = BertWordPieceTokenizer(\n            MODEL_PATHS[config.selected_model] + 'vocab.txt',\n            lowercase=config.lowercase,\n        )\n\n        tokens = {\n            'cls': tokenizer.token_to_id('[CLS]'),\n            'sep': tokenizer.token_to_id('[SEP]'),\n            'pad': tokenizer.token_to_id('[PAD]'),\n        }\n    \n    return tokenizer, tokens","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.023958Z","iopub.execute_input":"2021-06-02T01:46:06.024378Z","iopub.status.idle":"2021-06-02T01:46:06.03774Z","shell.execute_reply.started":"2021-06-02T01:46:06.024344Z","shell.execute_reply":"2021-06-02T01:46:06.036252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport json\nimport numpy as np\n\n\ndef load_text(id_, root=\"\"):\n    with open(os.path.join(root, id_ + \".json\")) as f:\n        text = json.load(f)\n    return text\n\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef locate_label_string(text, label):\n    \"\"\"\n    Finds the label in the text\n    \"\"\"\n    len_label = len(label) - 1\n\n    candidates_idx = [i for i, e in enumerate(text) if e == label[1]]\n    for idx in candidates_idx:\n        if \" \" + text[idx: idx + len_label] == label:\n            idx_start = idx\n            idx_end = idx + len_label\n            break\n\n    assert (\n        text[idx_start:idx_end] == label[1:]\n    ), f'\"{text[idx_start: idx_end]}\" instead of \"{label}\" in \"{text}\"'\n\n    char_targets = np.zeros(len(text))\n    char_targets[idx_start:idx_end] = 1\n\n    return idx_start, idx_end, char_targets\n\n\ndef locate_label_tokens(offsets, char_targets):\n    \"\"\"\n    Finds the tokens corresponding to the found labels\n    \"\"\"\n    target_idx = []\n    for idx, (offset1, offset2) in enumerate(offsets):\n        if sum(char_targets[offset1:offset2]) > 0:\n            target_idx.append(idx)\n\n    if not len(target_idx):\n        for idx, (offset1, offset2) in enumerate(offsets):\n            if sum(char_targets[offset1:offset2]) > 0:\n                target_idx.append(idx)\n\n    return target_idx[0], target_idx[-1]","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.043012Z","iopub.execute_input":"2021-06-02T01:46:06.043407Z","iopub.status.idle":"2021-06-02T01:46:06.061091Z","shell.execute_reply.started":"2021-06-02T01:46:06.043373Z","shell.execute_reply":"2021-06-02T01:46:06.059846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process sample","metadata":{}},{"cell_type":"code","source":"def process_data(\n    text,\n    label,\n    tokenizer,\n    tokens,\n    max_len=100,\n    model_name=\"bert\",\n):\n    \"\"\"\n    Prepares the data for the question answering task.\n    Adapted from Abishek's work on the Tweet Sentiment extraction competition, \n    check his work for more details !\n    \"\"\"\n    target_start, target_end = 0, 0\n    text = \" \" + \" \".join(str(text).split())\n    label = \" \" + \" \".join(str(label).split())\n\n    if label != \" \":\n        idx_start, idx_end, char_targets = locate_label_string(\n            text, label\n        )\n\n    tokenized = tokenizer.encode(text)\n    input_ids_text = tokenized.ids[1:-1]\n\n    # print(input_ids_text, len(input_ids_text))\n\n    offsets = tokenized.offsets[1:-1]\n\n    if label != \" \":\n        target_start, target_end = locate_label_tokens(offsets, char_targets)\n\n    if target_end >= max_len - 2:  # target is too far in the sentence, we crop its beginning.\n        n_tok_to_crop = target_start - max_len // 2\n        new_str_start = offsets[n_tok_to_crop][0]\n\n        input_ids_text = input_ids_text[n_tok_to_crop:]\n\n        offsets = [tuple(t) for t in np.array(offsets[n_tok_to_crop:]) - new_str_start]\n        text = text[new_str_start:]\n\n        target_start -= n_tok_to_crop\n        target_end -= n_tok_to_crop\n\n    input_ids = (\n        [tokens[\"cls\"]]\n        + input_ids_text[:max_len - 2]\n        + [tokens[\"sep\"]]\n    )\n\n    if \"roberta\" in model_name:\n        token_type_ids = [0] * len(input_ids)\n    else:\n        token_type_ids = [1] * len(input_ids)\n\n    text_offsets = [(0, 0)] + offsets[:max_len - 2] + [(0, 0)]\n\n    target_start += 1\n    target_end += 1\n\n    # target_end = min(target_end, max_len - 1)\n\n    assert len(input_ids) == len(token_type_ids) and len(input_ids) == len(text_offsets), (len(input_ids), len(text_offsets))  # noqa\n\n    padding_length = max_len - len(input_ids)\n    if padding_length > 0:\n        input_ids = input_ids + ([tokens[\"pad\"]] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        text_offsets = text_offsets + ([(0, 0)] * padding_length)\n\n    return {\n        \"ids\": input_ids,\n        \"token_type_ids\": token_type_ids,\n        \"targets_start\": target_start,\n        \"targets_end\": target_end,\n        \"text\": text,\n        \"label\": label,\n        \"offsets\": text_offsets,\n    }","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.065393Z","iopub.execute_input":"2021-06-02T01:46:06.066111Z","iopub.status.idle":"2021-06-02T01:46:06.088396Z","shell.execute_reply.started":"2021-06-02T01:46:06.066056Z","shell.execute_reply":"2021-06-02T01:46:06.086887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass ArticleDataset(Dataset):\n    \"\"\"\n    Dataset for inference. \n    \"\"\"\n    def __init__(\n        self,\n        id_,\n        tokenizer,\n        tokens,\n        max_len=512,\n        words_per_split=300,\n        margin=10,\n        model_name=\"bert\",\n        root=\"\"\n    ):\n        self.tokens = tokens\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.model_name = model_name\n        self.words_per_split = words_per_split\n        self.margin = margin\n\n        self.article = load_text(id_, root=root)\n        \n        self.texts = self.article_to_texts()\n\n    def __len__(self):\n        return len(self.texts)\n    \n    def article_to_texts(self):\n        \"\"\"\n        Each article is divided into sections, \n        and then into subsets of self.words_per_split words\n        \"\"\"\n        texts = []\n        for section in self.article:\n            clean_section = clean_text(section['text']).split(' ')[:5000]  # only keep first 5k words\n            \n            for i in range(len(clean_section) // self.words_per_split + 1):\n                start = max(0, self.words_per_split * i - self.margin)\n                end = self.words_per_split * (i + 1) + self.margin\n                text = \" \".join(clean_section[start: end])\n                texts.append(text)\n            \n        return texts\n\n    def __getitem__(self, idx):\n        data = process_data(\n            self.texts[idx],\n            \"\",\n            self.tokenizer,\n            self.tokens,\n            max_len=self.max_len,\n            model_name=self.model_name,\n        )\n\n        return {\n            \"ids\": torch.tensor(data[\"ids\"], dtype=torch.long),\n            \"token_type_ids\": torch.tensor(data[\"token_type_ids\"], dtype=torch.long),\n            \"target_start\": torch.tensor(data[\"targets_start\"], dtype=torch.long),\n            \"target_end\": torch.tensor(data[\"targets_end\"], dtype=torch.long),\n            \"text\": data[\"text\"],\n            \"label\": data[\"label\"],\n            \"offsets\": torch.tensor(data[\"offsets\"], dtype=torch.long),\n        }\n","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.090485Z","iopub.execute_input":"2021-06-02T01:46:06.091292Z","iopub.status.idle":"2021-06-02T01:46:06.11429Z","shell.execute_reply.started":"2021-06-02T01:46:06.091249Z","shell.execute_reply":"2021-06-02T01:46:06.11317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"from transformers import BertModel, BertConfig\n\nTRANSFORMERS = {   \n    \"bert-base-uncased\": (BertModel, \"bert-base-uncased\", BertConfig),\n}\n\n\nclass QATransformer(nn.Module):\n    \"\"\"\n    Simple model for Question Answering\n    \"\"\"\n    def __init__(self, model):\n        super().__init__()\n        self.name = model\n\n        self.pad_idx = 1 if \"roberta\" in self.name else 0\n\n        model_class, _, config_class = TRANSFORMERS[model]\n\n        try:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'bert_config.json')\n        except:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'config.json')\n        config.output_hidden_states = True\n\n        self.transformer =  model_class(config)\n\n        self.nb_features = self.transformer.pooler.dense.out_features\n\n        self.logits = nn.Sequential(\n            nn.Linear(self.nb_features, self.nb_features),\n            nn.Tanh(),\n            nn.Linear(self.nb_features, 2),\n        )\n\n    def forward(self, tokens, token_type_ids):\n        \"\"\"\n        Usual torch forward function\n\n        Arguments:\n            tokens {torch tensor} -- Sentence tokens\n            token_type_ids {torch tensor} -- Sentence tokens ids\n        \"\"\"\n\n        hidden_states = self.transformer(\n            tokens,\n            attention_mask=(tokens != self.pad_idx).long(),\n            token_type_ids=token_type_ids,\n        )[-1]\n\n        features = hidden_states[-1]\n        logits = self.logits(features)\n\n        start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]\n\n        return start_logits, end_logits","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.115894Z","iopub.execute_input":"2021-06-02T01:46:06.116232Z","iopub.status.idle":"2021-06-02T01:46:06.357708Z","shell.execute_reply.started":"2021-06-02T01:46:06.116199Z","shell.execute_reply":"2021-06-02T01:46:06.356355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\n","metadata":{}},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=True)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.359241Z","iopub.execute_input":"2021-06-02T01:46:06.359601Z","iopub.status.idle":"2021-06-02T01:46:06.366751Z","shell.execute_reply.started":"2021-06-02T01:46:06.359565Z","shell.execute_reply":"2021-06-02T01:46:06.365707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom torch.utils.data import DataLoader\n\n\ndef predict(model, dataset, batch_size=32):\n    \"\"\"\n    Usual predict torch function\n\n    Arguments:\n        model {torch model} -- Model to predict with\n        dataset {torch dataset} -- Dataset to get predictions from\n\n    Keyword Arguments:\n        batch_size {int} -- Batch size (default: {32})\n\n    Returns:\n        numpy array -- Predictions\n    \"\"\"\n\n    model.eval()\n    start_probas = []\n    end_probas = []\n\n    loader = DataLoader(\n        dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS\n    )\n\n    with torch.no_grad():\n        for data in loader:\n            ids, token_type_ids = data[\"ids\"], data[\"token_type_ids\"]\n\n            start_logits, end_logits = model(\n                ids.cuda(), token_type_ids.cuda()\n            )\n\n            start_probs = torch.softmax(start_logits, dim=1).cpu().detach().numpy()\n            end_probs = torch.softmax(end_logits, dim=1).cpu().detach().numpy()\n\n            for s, e in zip(start_probs, end_probs):\n                start_probas.append(list(s))\n                end_probas.append(list(e))\n\n    return start_probas, end_probas","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.368256Z","iopub.execute_input":"2021-06-02T01:46:06.368594Z","iopub.status.idle":"2021-06-02T01:46:06.392282Z","shell.execute_reply.started":"2021-06-02T01:46:06.36856Z","shell.execute_reply":"2021-06-02T01:46:06.391167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicted strings from probas","metadata":{}},{"cell_type":"code","source":"def get_string_from_idx(text, idx_start, idx_end, offsets):\n    \"\"\"\n    Uses the offsets to retrieve the predicted string based on the start and end indices\n    \"\"\"\n    if idx_end < idx_start:\n        idx_end = idx_start\n\n    predicted_string = \"\"\n    for i in range(idx_start, idx_end + 1):\n        predicted_string += text[offsets[i][0]: offsets[i][1]]\n        if i + 1 < len(offsets) and offsets[i][1] < offsets[i + 1][0]:\n            predicted_string += \" \"\n\n    return predicted_string\n\n\ndef get_pred_from_probas(dataset, start_probas, end_probas, threshold=0.):\n    preds = []\n    for i in range(len(dataset)):\n        if start_probas[i].max() > threshold or end_probas[i].max() > threshold:\n            start_idx = np.argmax(start_probas[i])\n            end_idx = np.argmax(end_probas[i])\n            if start_idx < end_idx and end_idx - start_idx < 10:\n                # print(start_idx, end_idx)\n                data = dataset[i]\n                preds.append(get_string_from_idx(data[\"text\"], start_idx, end_idx, data[\"offsets\"]))\n\n    return preds","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.396546Z","iopub.execute_input":"2021-06-02T01:46:06.396901Z","iopub.status.idle":"2021-06-02T01:46:06.41494Z","shell.execute_reply.started":"2021-06-02T01:46:06.39687Z","shell.execute_reply":"2021-06-02T01:46:06.413826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## $k$-fold","metadata":{}},{"cell_type":"code","source":"def post_process(preds):\n    \"\"\"\n    Naive processing of prediction : \n    Remove duplicates and convert to expected format.\n    \"\"\"\n    preds = np.unique(preds)\n    return \"|\".join(preds)\n\n\ndef k_fold_inference(config, df, tokenizer, tokens, weights, threshold=0.):\n    models = []\n    for w in weights:\n        model = QATransformer(config.selected_model).cuda()\n        model.zero_grad()\n        load_model_weights(model, w)\n        models.append(model)\n\n    preds = []\n    for text_id in tqdm(df['Id']):\n\n        dataset = ArticleDataset(\n            text_id,\n            tokenizer,\n            tokens,\n            max_len=512,\n            model_name=\"bert\",\n            root=DATA_PATH_TEST\n        )\n\n        start_probas, end_probas = [], []\n        for model in models:\n            start_proba, end_proba = predict(\n                model, \n                dataset, \n                batch_size=config.batch_size_val, \n            )\n            start_probas.append(start_proba)\n            end_probas.append(end_proba)\n\n        start_probas = np.mean(start_probas, 0)\n        end_probas = np.mean(end_probas, 0)\n        \n        # here can do some FOPP\n        pred = get_pred_from_probas(dataset, start_probas, end_probas, threshold=threshold)\n        preds.append(post_process(pred))\n            \n    return preds","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:06.416272Z","iopub.execute_input":"2021-06-02T01:46:06.416587Z","iopub.status.idle":"2021-06-02T01:46:06.436908Z","shell.execute_reply.started":"2021-06-02T01:46:06.416556Z","shell.execute_reply":"2021-06-02T01:46:06.435878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"config = Config\ndf = pd.read_csv(DATA_PATH + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.438078Z","iopub.execute_input":"2021-06-02T01:46:06.43837Z","iopub.status.idle":"2021-06-02T01:46:06.469145Z","shell.execute_reply.started":"2021-06-02T01:46:06.438342Z","shell.execute_reply":"2021-06-02T01:46:06.468021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer, tokens = create_tokenizer_and_tokens(config)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.47287Z","iopub.execute_input":"2021-06-02T01:46:06.473197Z","iopub.status.idle":"2021-06-02T01:46:06.530424Z","shell.execute_reply.started":"2021-06-02T01:46:06.47317Z","shell.execute_reply":"2021-06-02T01:46:06.529281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = ArticleDataset(\n    df['Id'][0],\n    tokenizer,\n    tokens,\n    max_len=512,\n    model_name=\"bert\",\n    root=DATA_PATH_TEST,\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.532206Z","iopub.execute_input":"2021-06-02T01:46:06.532553Z","iopub.status.idle":"2021-06-02T01:46:06.563192Z","shell.execute_reply.started":"2021-06-02T01:46:06.53252Z","shell.execute_reply":"2021-06-02T01:46:06.561709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = [sorted(glob.glob(CP_PATH + \"*.pt\"))[-1]] # -> list(model_paths)\nweights","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.566029Z","iopub.execute_input":"2021-06-02T01:46:06.566369Z","iopub.status.idle":"2021-06-02T01:46:06.588903Z","shell.execute_reply.started":"2021-06-02T01:46:06.566334Z","shell.execute_reply":"2021-06-02T01:46:06.587631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_model = k_fold_inference(\n    config,\n    df,\n    tokenizer,\n    tokens,\n    weights,\n    threshold=THRESHOLD,\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:06.590566Z","iopub.execute_input":"2021-06-02T01:46:06.591019Z","iopub.status.idle":"2021-06-02T01:46:09.197091Z","shell.execute_reply.started":"2021-06-02T01:46:06.590971Z","shell.execute_reply":"2021-06-02T01:46:09.194423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Match","metadata":{}},{"cell_type":"code","source":"train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.198309Z","iopub.status.idle":"2021-06-02T01:46:09.198843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\n\ndef read_json_pub(filename, train_data_path=train_files_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    \n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.20317Z","iopub.status.idle":"2021-06-02T01:46:09.204086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if HOW != 'QA_ONLY':\n    sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    adnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\n\n    literal_preds = []\n    to_append = []\n    for index, row in tqdm(sample_submission.iterrows()):\n        to_append = [row['Id'],'']\n        large_string = str(read_json_pub(row['Id'], test_files_path))\n        clean_string = text_cleaning(large_string)\n        for index, row2 in adnl_govt_labels.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n                    to_append[1] = to_append[1] + '|' + clean_text(query_string)\n                if to_append[1] == '':\n                    to_append[1] = clean_text(query_string)\n        literal_preds.append(*to_append[1:])\n    \n    preds_naive = literal_preds","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.205536Z","iopub.status.idle":"2021-06-02T01:46:09.206202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge","metadata":{}},{"cell_type":"code","source":"def merge_preds(preds_naive, preds_model):\n    preds = []\n    for i in range(len(preds_naive)):\n        pred_naive = preds_naive[i].split('|')\n        pred_model = preds_model[i].split('|')\n        \n        pred_model_kept = []\n        for pred_m in pred_model:\n            kept = True\n            for pred_n in pred_naive:\n                if pred_m in pred_n or pred_n in pred_m:\n                    kept = False\n            \n            if kept:\n                pred_model_kept.append(pred_m)\n            else:\n                pass\n#                 print(f'Removed prediction {pred_m}')\n            \n        preds.append(\"|\".join(pred_naive + pred_model_kept))\n    return preds","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-02T01:46:09.207371Z","iopub.status.idle":"2021-06-02T01:46:09.207919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = merge_preds(preds_naive, preds_model)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.209031Z","iopub.status.idle":"2021-06-02T01:46:09.209552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.210866Z","iopub.status.idle":"2021-06-02T01:46:09.211364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\n\n\nif HOW == 'QA_ONLY':\n    final_predictions = preds_model\n    \nelif HOW == 'LAM':\n    for pred_match, perd_mlm in tqdm(zip(preds_naive, preds_model)):\n        if pred_match:\n            labels = pred_match.split('|')\n            \n            # literal_preds + pred_mlm_labels\n            if perd_mlm:\n                filtered_labels = labels\n                labels_mlm = perd_mlm.split('|')\n                for label_mlm in labels_mlm:\n                    if all(jaccard_similarity(label_mlm, got_label) < LAM_FL_TH for got_label in labels):\n                        filtered_labels.append(label_mlm)\n                        \n            # literal_preds\n            else: filtered_labels = labels\n                \n        # pred_mlm_labels\n        elif perd_mlm:\n            filtered_labels = perd_mlm.split('|')\n        \n        # ''\n        else:\n            filtered_labels = []\n            \n        final_predictions.append('|'.join(filtered_labels))\n    print(f'final_predictions[:4] w/ LAM_FL_TH{LAM_FL_TH}:')\n\nelif HOW == 'ORIGINAL':\n    for literal_match, mlm_pred in zip(preds_naive, preds_model):\n        if literal_match:\n            final_predictions.append(literal_match)\n        else:\n            final_predictions.append(mlm_pred)\n            \n\npreds = final_predictions\npreds[:4]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.212471Z","iopub.status.idle":"2021-06-02T01:46:09.212998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = preds_model","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.214284Z","iopub.status.idle":"2021-06-02T01:46:09.214801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"df['PredictionString'] = preds\n\ndf.to_csv('submission.csv', index=False)\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.215894Z","iopub.status.idle":"2021-06-02T01:46:09.216359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'HOW = {HOW}')\nprint(f'THRESHOLD = {THRESHOLD}')\nif HOW == 'LAM': print(f'LAM_FL_TH = {LAM_FL_TH}')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T01:46:09.217499Z","iopub.status.idle":"2021-06-02T01:46:09.218026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|   | CV | LB |\n| --- | --- | --- |\n| QA w/ External_Datasets_Matching |   | 0.548 |\n| QA LAM(bug) |   | 0.160 |\n| QA LAM |   | 0.400 |\n| QA only |   | 0.299 |\n| QA only 0.6 |   | 0.313 |\n| QA only 0.7 |   | 0.330 |\n| QA only 0.8 |   | 0.343 |\n| QA only 0.9 |   | 0.364 |\n| QA only 0.95 |   | 0.373 |\n| QA only 0.975 |   | 0.376 |\n| **QA only 0.995** |   | **0.378** |\n| QA only 0.999 |   |   |\n| QAv1 Pseudo only 0.8 |   | 0.320 |\n| QAv10 Pseudo External all_length only 0.9 |   | 0.300 |","metadata":{}}]}