{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🤗 Bert for Question Answering Baseline: Inference\n\nThis code is adapted from my work in the Tweet Sentiment Extraction Competition.\n\nIt tackles the task as a Question Answering one, where the question is implicit and can be understood as : \"Which datasets are mentionned ?\"\n\n\nThe approach is quite naïve and has a lot of flaws. Feel free to ask any question in the comments.\n\nTraining Kernel : https://www.kaggle.com/theoviel/bert-for-question-answering-baseline-training","metadata":{}},{"cell_type":"markdown","source":"# Initialization","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport glob\nimport json\nimport torch\nimport datetime\nimport tokenizers\nimport numpy as np\nimport transformers\nimport pandas as pd\nimport torch.nn as nn\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tokenizers import *\nfrom functools import partial\nfrom tqdm.notebook import tqdm\nfrom torch.nn import functional as F\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Params","metadata":{}},{"cell_type":"code","source":"SEED = 2020\n\nDATA_PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\nDATA_PATH_TRAIN = DATA_PATH + 'train/'\nDATA_PATH_TEST = DATA_PATH + 'test/'\n\nCP_PATH = '../input/coleridge-bert-qa/'\n\nNUM_WORKERS = 4\n\nVOCABS = {\n    \"bert-base-uncased\": \"../input/vocabs/bert-base-uncased-vocab.txt\",\n}\n\nMODEL_PATHS = {\n    'bert-base-uncased': '../input/bertconfigs/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/',\n    'bert-large-uncased-whole-word-masking-finetuned-squad': '../input/bertconfigs/wwm_uncased_L-24_H-1024_A-16/wwm_uncased_L-24_H-1024_A-16/',\n    'albert-large-v2': '../input/albert-configs/albert-large-v2/albert-large-v2/',\n    'albert-base-v2': '../input/albert-configs/albert-base-v2/albert-base-v2/',\n    'distilbert': '../input/albert-configs/distilbert/distilbert/',\n}","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    # General\n    k = 5\n    seed = 2021\n\n    # Texts\n    max_len = 256\n    \n    # Architecture\n    selected_model = \"bert-base-uncased\"\n    lowercase = True\n    \n    # Training\n    batch_size = 16\n    batch_size_val = batch_size * 2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"markdown","source":"## Tokenizer","metadata":{}},{"cell_type":"code","source":"class EncodedText:\n    def __init__(self, ids, offsets):\n        self.ids = ids\n        self.offsets = offsets\n\n\ndef create_tokenizer_and_tokens(config):\n    if \"roberta\" in config.selected_model:\n        raise NotImplementedError\n        \n    elif \"albert\" in config.selected_model:\n        raise NotImplementedError\n        \n    else:\n        tokenizer = BertWordPieceTokenizer(\n            MODEL_PATHS[config.selected_model] + 'vocab.txt',\n            lowercase=config.lowercase,\n        )\n\n        tokens = {\n            'cls': tokenizer.token_to_id('[CLS]'),\n            'sep': tokenizer.token_to_id('[SEP]'),\n            'pad': tokenizer.token_to_id('[PAD]'),\n        }\n    \n    return tokenizer, tokens","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport json\nimport numpy as np\n\n\ndef load_text(id_, root=\"\"):\n    with open(os.path.join(root, id_ + \".json\")) as f:\n        text = json.load(f)\n    return text\n\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef locate_label_string(text, label):\n    \"\"\"\n    Finds the label in the text\n    \"\"\"\n    len_label = len(label) - 1\n\n    candidates_idx = [i for i, e in enumerate(text) if e == label[1]]\n    for idx in candidates_idx:\n        if \" \" + text[idx: idx + len_label] == label:\n            idx_start = idx\n            idx_end = idx + len_label\n            break\n\n    assert (\n        text[idx_start:idx_end] == label[1:]\n    ), f'\"{text[idx_start: idx_end]}\" instead of \"{label}\" in \"{text}\"'\n\n    char_targets = np.zeros(len(text))\n    char_targets[idx_start:idx_end] = 1\n\n    return idx_start, idx_end, char_targets\n\n\ndef locate_label_tokens(offsets, char_targets):\n    \"\"\"\n    Finds the tokens corresponding to the found labels\n    \"\"\"\n    target_idx = []\n    for idx, (offset1, offset2) in enumerate(offsets):\n        if sum(char_targets[offset1:offset2]) > 0:\n            target_idx.append(idx)\n\n    if not len(target_idx):\n        for idx, (offset1, offset2) in enumerate(offsets):\n            if sum(char_targets[offset1:offset2]) > 0:\n                target_idx.append(idx)\n\n    return target_idx[0], target_idx[-1]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process sample","metadata":{}},{"cell_type":"code","source":"def process_data(\n    text,\n    label,\n    tokenizer,\n    tokens,\n    max_len=100,\n    model_name=\"bert\",\n):\n    \"\"\"\n    Prepares the data for the question answering task.\n    Adapted from Abishek's work on the Tweet Sentiment extraction competition, \n    check his work for more details !\n    \"\"\"\n    target_start, target_end = 0, 0\n    text = \" \" + \" \".join(str(text).split())\n    label = \" \" + \" \".join(str(label).split())\n\n    if label != \" \":\n        idx_start, idx_end, char_targets = locate_label_string(\n            text, label\n        )\n\n    tokenized = tokenizer.encode(text)\n    input_ids_text = tokenized.ids[1:-1]\n\n    # print(input_ids_text, len(input_ids_text))\n\n    offsets = tokenized.offsets[1:-1]\n\n    if label != \" \":\n        target_start, target_end = locate_label_tokens(offsets, char_targets)\n\n    if target_end >= max_len - 2:  # target is too far in the sentence, we crop its beginning.\n        n_tok_to_crop = target_start - max_len // 2\n        new_str_start = offsets[n_tok_to_crop][0]\n\n        input_ids_text = input_ids_text[n_tok_to_crop:]\n\n        offsets = [tuple(t) for t in np.array(offsets[n_tok_to_crop:]) - new_str_start]\n        text = text[new_str_start:]\n\n        target_start -= n_tok_to_crop\n        target_end -= n_tok_to_crop\n\n    input_ids = (\n        [tokens[\"cls\"]]\n        + input_ids_text[:max_len - 2]\n        + [tokens[\"sep\"]]\n    )\n\n    if \"roberta\" in model_name:\n        token_type_ids = [0] * len(input_ids)\n    else:\n        token_type_ids = [1] * len(input_ids)\n\n    text_offsets = [(0, 0)] + offsets[:max_len - 2] + [(0, 0)]\n\n    target_start += 1\n    target_end += 1\n\n    # target_end = min(target_end, max_len - 1)\n\n    assert len(input_ids) == len(token_type_ids) and len(input_ids) == len(text_offsets), (len(input_ids), len(text_offsets))  # noqa\n\n    padding_length = max_len - len(input_ids)\n    if padding_length > 0:\n        input_ids = input_ids + ([tokens[\"pad\"]] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        text_offsets = text_offsets + ([(0, 0)] * padding_length)\n\n    return {\n        \"ids\": input_ids,\n        \"token_type_ids\": token_type_ids,\n        \"targets_start\": target_start,\n        \"targets_end\": target_end,\n        \"text\": text,\n        \"label\": label,\n        \"offsets\": text_offsets,\n    }","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass ArticleDataset(Dataset):\n    \"\"\"\n    Dataset for inference. \n    \"\"\"\n    def __init__(\n        self,\n        id_,\n        tokenizer,\n        tokens,\n        max_len=512,\n        words_per_split=300,\n        margin=10,\n        model_name=\"bert\",\n        root=\"\"\n    ):\n        self.tokens = tokens\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.model_name = model_name\n        self.words_per_split = words_per_split\n        self.margin = margin\n\n        self.article = load_text(id_, root=root)\n        \n        self.texts = self.article_to_texts()\n\n    def __len__(self):\n        return len(self.texts)\n    \n    def article_to_texts(self):\n        \"\"\"\n        Each article is divided into sections, \n        and then into subsets of self.words_per_split words\n        \"\"\"\n        texts = []\n        for section in self.article:\n            clean_section = clean_text(section['text']).split(' ')[:5000]  # only keep first 5k words\n            \n            for i in range(len(clean_section) // self.words_per_split + 1):\n                start = max(0, self.words_per_split * i - self.margin)\n                end = self.words_per_split * (i + 1) + self.margin\n                text = \" \".join(clean_section[start: end])\n                texts.append(text)\n            \n        return texts\n\n    def __getitem__(self, idx):\n        data = process_data(\n            self.texts[idx],\n            \"\",\n            self.tokenizer,\n            self.tokens,\n            max_len=self.max_len,\n            model_name=self.model_name,\n        )\n\n        return {\n            \"ids\": torch.tensor(data[\"ids\"], dtype=torch.long),\n            \"token_type_ids\": torch.tensor(data[\"token_type_ids\"], dtype=torch.long),\n            \"target_start\": torch.tensor(data[\"targets_start\"], dtype=torch.long),\n            \"target_end\": torch.tensor(data[\"targets_end\"], dtype=torch.long),\n            \"text\": data[\"text\"],\n            \"label\": data[\"label\"],\n            \"offsets\": torch.tensor(data[\"offsets\"], dtype=torch.long),\n        }\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"from transformers import BertModel, BertConfig\n\nTRANSFORMERS = {   \n    \"bert-base-uncased\": (BertModel, \"bert-base-uncased\", BertConfig),\n}\n\n\nclass QATransformer(nn.Module):\n    \"\"\"\n    Simple model for Question Answering\n    \"\"\"\n    def __init__(self, model):\n        super().__init__()\n        self.name = model\n\n        self.pad_idx = 1 if \"roberta\" in self.name else 0\n\n        model_class, _, config_class = TRANSFORMERS[model]\n\n        try:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'bert_config.json')\n        except:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'config.json')\n        config.output_hidden_states = True\n\n        self.transformer =  model_class(config)\n\n        self.nb_features = self.transformer.pooler.dense.out_features\n\n        self.logits = nn.Sequential(\n            nn.Linear(self.nb_features, self.nb_features),\n            nn.Tanh(),\n            nn.Linear(self.nb_features, 2),\n        )\n\n    def forward(self, tokens, token_type_ids):\n        \"\"\"\n        Usual torch forward function\n\n        Arguments:\n            tokens {torch tensor} -- Sentence tokens\n            token_type_ids {torch tensor} -- Sentence tokens ids\n        \"\"\"\n\n        hidden_states = self.transformer(\n            tokens,\n            attention_mask=(tokens != self.pad_idx).long(),\n            token_type_ids=token_type_ids,\n        )[-1]\n\n        features = hidden_states[-1]\n        logits = self.logits(features)\n\n        start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]\n\n        return start_logits, end_logits","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\n","metadata":{}},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=True)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom torch.utils.data import DataLoader\n\n\ndef predict(model, dataset, batch_size=32):\n    \"\"\"\n    Usual predict torch function\n\n    Arguments:\n        model {torch model} -- Model to predict with\n        dataset {torch dataset} -- Dataset to get predictions from\n\n    Keyword Arguments:\n        batch_size {int} -- Batch size (default: {32})\n\n    Returns:\n        numpy array -- Predictions\n    \"\"\"\n\n    model.eval()\n    start_probas = []\n    end_probas = []\n\n    loader = DataLoader(\n        dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS\n    )\n\n    with torch.no_grad():\n        for data in loader:\n            ids, token_type_ids = data[\"ids\"], data[\"token_type_ids\"]\n\n            start_logits, end_logits = model(\n                ids.cuda(), token_type_ids.cuda()\n            )\n\n            start_probs = torch.softmax(start_logits, dim=1).cpu().detach().numpy()\n            end_probs = torch.softmax(end_logits, dim=1).cpu().detach().numpy()\n\n            for s, e in zip(start_probs, end_probs):\n                start_probas.append(list(s))\n                end_probas.append(list(e))\n\n    return start_probas, end_probas","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicted strings from probas","metadata":{}},{"cell_type":"code","source":"def get_string_from_idx(text, idx_start, idx_end, offsets):\n    \"\"\"\n    Uses the offsets to retrieve the predicted string based on the start and end indices\n    \"\"\"\n    if idx_end < idx_start:\n        idx_end = idx_start\n\n    predicted_string = \"\"\n    for i in range(idx_start, idx_end + 1):\n        predicted_string += text[offsets[i][0]: offsets[i][1]]\n        if i + 1 < len(offsets) and offsets[i][1] < offsets[i + 1][0]:\n            predicted_string += \" \"\n\n    return predicted_string\n\n\ndef get_pred_from_probas(dataset, start_probas, end_probas, threshold=0.):\n    preds = []\n    for i in range(len(dataset)):\n        if start_probas[i].max() > threshold or end_probas[i].max() > threshold:\n            start_idx = np.argmax(start_probas[i])\n            end_idx = np.argmax(end_probas[i])\n            if start_idx < end_idx and end_idx - start_idx < 10:\n                # print(start_idx, end_idx)\n                data = dataset[i]\n                preds.append(get_string_from_idx(data[\"text\"], start_idx, end_idx, data[\"offsets\"]))\n\n    return preds","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## $k$-fold","metadata":{}},{"cell_type":"code","source":"def post_process(preds):\n    \"\"\"\n    Naive processing of prediction : \n    Remove duplicates and convert to expected format.\n    \"\"\"\n    preds = np.unique(preds)\n    return \"|\".join(preds)\n\n\ndef k_fold_inference(config, df, tokenizer, tokens, weights, threshold=0.):\n    models = []\n    for w in weights:\n        model = QATransformer(config.selected_model).cuda()\n        model.zero_grad()\n        load_model_weights(model, w)\n        models.append(model)\n\n    preds = []\n    for text_id in tqdm(df['Id']):\n\n        dataset = ArticleDataset(\n            text_id,\n            tokenizer,\n            tokens,\n            max_len=512,\n            model_name=\"bert\",\n            root=DATA_PATH_TEST\n        )\n\n        start_probas, end_probas = [], []\n        for model in models:\n            start_proba, end_proba = predict(\n                model, \n                dataset, \n                batch_size=config.batch_size_val, \n            )\n            start_probas.append(start_proba)\n            end_probas.append(end_proba)\n\n        start_probas = np.mean(start_probas, 0)\n        end_probas = np.mean(end_probas, 0)\n\n        pred = get_pred_from_probas(dataset, start_probas, end_probas, threshold=threshold)\n        preds.append(post_process(pred))\n            \n    return preds","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"config = Config\ndf = pd.read_csv(DATA_PATH + 'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer, tokens = create_tokenizer_and_tokens(config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = ArticleDataset(\n    df['Id'][0],\n    tokenizer,\n    tokens,\n    max_len=512,\n    model_name=\"bert\",\n    root=DATA_PATH_TEST,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = sorted(glob.glob(CP_PATH + \"*.pt\"))[:1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_model = k_fold_inference(\n    config,\n    df,\n    tokenizer,\n    tokens,\n    weights,\n    threshold=0.5,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge with 0.702 notebook\n> https://www.kaggle.com/prashansdixit/coleridge-initiative-eda-baseline-model","metadata":{}},{"cell_type":"markdown","source":"## Compute","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, root=DATA_PATH_TRAIN, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(root, (filename +'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\ntemp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub['text'] = sample_sub['Id'].apply(partial(read_append_return, root=DATA_PATH_TEST))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_naive = []\nfor index, row in sample_sub.iterrows():\n    sample_text = row['text']\n    row_id = row['Id']\n    \n#     temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n#     cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    cleaned_labels = []\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n\n    cleaned_labels = set([clean_text(x) for x in cleaned_labels])\n    \n    preds_naive.append('|'.join(cleaned_labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge","metadata":{}},{"cell_type":"code","source":"def merge_preds(preds_naive, preds_model):\n    preds = []\n    for i in range(len(preds_naive)):\n        pred_naive = preds_naive[i].split('|')\n        pred_model = preds_model[i].split('|')\n        \n        pred_model_kept = []\n        for pred_m in pred_model:\n            kept = True\n            for pred_n in pred_naive:\n                if pred_m in pred_n or pred_n in pred_m:\n                    kept = False\n            \n            if kept:\n                pred_model_kept.append(pred_m)\n            else:\n                pass\n#                 print(f'Removed prediction {pred_m}')\n            \n        preds.append(\"|\".join(pred_naive + pred_model_kept))\n    return preds","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = merge_preds(preds_naive, preds_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"code","source":"df['PredictionString'] = preds\n\ndf.to_csv('submission.csv', index=False)\n\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks for reading !\n\nHopefully this work comes helpful in beating the public LB baselines...\n\nDon't forget to upvote :)","metadata":{}}]}