{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initialization","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport glob\nimport json\nimport torch\nimport tokenizers\nimport numpy as np\nimport transformers\nimport pandas as pd\nimport torch.nn as nn\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tokenizers import *\nfrom transformers import *\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:29.237768Z","iopub.execute_input":"2021-06-14T21:02:29.238152Z","iopub.status.idle":"2021-06-14T21:02:38.694652Z","shell.execute_reply.started":"2021-06-14T21:02:29.238076Z","shell.execute_reply":"2021-06-14T21:02:38.693773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Params","metadata":{}},{"cell_type":"code","source":"SEED = 2020\n\nDATA_PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\nDATA_PATH_TRAIN = DATA_PATH + 'train/'\nDATA_PATH_TEST = DATA_PATH + 'test/'\n\nCP_PATH = '../input/coleridge-bert-qa/'\n\nNUM_WORKERS = 4\n\nVOCABS = {\n    \"bert-base-uncased\": \"../input/vocabs/bert-base-uncased-vocab.txt\",\n    \"roberta-base\": \"../input/vocabs/roberta-base-vocab.json\",\n}\n\nMERGES = {\n    \"roberta-base\": \"../input/vocabs/roberta-base-merges.txt\"\n}\n\nMODEL_PATHS = {\n    'bert-base-uncased': '../input/bertconfigs/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/',\n    'bert-large-uncased-whole-word-masking-finetuned-squad': '../input/bertconfigs/wwm_uncased_L-24_H-1024_A-16/wwm_uncased_L-24_H-1024_A-16/',\n    'albert-large-v2': '../input/albert-configs/albert-large-v2/albert-large-v2/',\n    'albert-base-v2': '../input/albert-configs/albert-base-v2/albert-base-v2/',\n    'distilbert': '../input/albert-configs/distilbert/distilbert/',\n    'roberta-base': '../input/robertabaseconf/',\n}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.699082Z","iopub.execute_input":"2021-06-14T21:02:38.699343Z","iopub.status.idle":"2021-06-14T21:02:38.706998Z","shell.execute_reply.started":"2021-06-14T21:02:38.699317Z","shell.execute_reply":"2021-06-14T21:02:38.705957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=True)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model\n\n\ndef seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results.\n    Args:\n        seed (int): Number of the seed.\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n\nclass Config:\n    \"\"\"\n    Placeholder to load a config from a saved json\n    \"\"\"\n    def __init__(self, dic):\n        for k, v in dic.items():\n            setattr(self, k, v)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.710155Z","iopub.execute_input":"2021-06-14T21:02:38.710437Z","iopub.status.idle":"2021-06-14T21:02:38.721381Z","shell.execute_reply.started":"2021-06-14T21:02:38.710410Z","shell.execute_reply":"2021-06-14T21:02:38.720744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"markdown","source":"## Tokenizer","metadata":{}},{"cell_type":"code","source":"def create_tokenizer_and_tokens(config):\n    if \"roberta\" in config.selected_model:\n        tokenizer = ByteLevelBPETokenizer(\n            vocab=VOCABS[config.selected_model],\n            merges=MERGES[config.selected_model],\n            lowercase=config.lowercase,\n            add_prefix_space=True,\n        )\n\n        tokens = {\n            \"cls\": tokenizer.token_to_id(\"<s>\"),\n            \"sep\": tokenizer.token_to_id(\"</s>\"),\n            \"pad\": tokenizer.token_to_id(\"<pad>\"),\n        }\n\n    elif \"albert\" in config.selected_model:\n        raise NotImplementedError\n\n    else:\n        tokenizer = BertWordPieceTokenizer(\n            VOCABS[config.selected_model],\n            lowercase=config.lowercase,\n        )\n\n        tokens = {\n            \"cls\": tokenizer.token_to_id(\"[CLS]\"),\n            \"sep\": tokenizer.token_to_id(\"[SEP]\"),\n            \"pad\": tokenizer.token_to_id(\"[PAD]\"),\n        }\n\n    return tokenizer, tokens\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.722501Z","iopub.execute_input":"2021-06-14T21:02:38.722890Z","iopub.status.idle":"2021-06-14T21:02:38.735932Z","shell.execute_reply.started":"2021-06-14T21:02:38.722854Z","shell.execute_reply":"2021-06-14T21:02:38.735174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process sample","metadata":{}},{"cell_type":"code","source":"def load_text(id_, root=\"\"):\n    with open(os.path.join(root, id_ + \".json\")) as f:\n        text = json.load(f)\n    return text\n\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef remove_spaces(txt):\n    txt = re.sub('\\n', ' ', txt)\n    txt = re.sub('\\t', ' ', txt)\n    txt = re.sub(r'\\s+', ' ', txt)\n    return txt.strip()\n\n\ndef process_data_inf(\n    text,\n    tokenizer,\n    tokens,\n    max_len=512,\n    model_name=\"bert\",\n):\n    text = \" \" + \" \".join(str(text).split())\n    tokenized = tokenizer.encode(text)\n\n    input_ids = tokenized.ids\n    offsets = tokenized.offsets\n\n    if len(input_ids) > max_len - 2:\n        input_ids = input_ids[:max_len - 2]\n        offsets = offsets[:max_len - 2]\n\n    input_ids = [tokens[\"cls\"]] + input_ids + [tokens[\"sep\"]]\n    offsets = [(0, 0)] + offsets + [(0, 0)]\n\n    assert len(input_ids) <= max_len\n    assert len(input_ids) == len(offsets)\n\n    if \"roberta\" in model_name:\n        token_type_ids = [0] * len(input_ids)\n    else:\n        token_type_ids = [1] * len(input_ids)\n\n    padding_length = max_len - len(input_ids)\n    if padding_length > 0:\n        input_ids += [tokens[\"pad\"]] * padding_length\n        token_type_ids += [0] * padding_length\n        offsets += [(0, 0)] * padding_length\n\n    return {\n        \"ids\": input_ids,\n        \"token_type_ids\": token_type_ids,\n        \"text\": text,\n        \"offsets\": offsets,\n    }\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.740650Z","iopub.execute_input":"2021-06-14T21:02:38.740984Z","iopub.status.idle":"2021-06-14T21:02:38.755316Z","shell.execute_reply.started":"2021-06-14T21:02:38.740957Z","shell.execute_reply":"2021-06-14T21:02:38.754416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class ArticleDataset(Dataset):\n    def __init__(\n        self,\n        id_,\n        tokenizer,\n        tokens,\n        max_len=512,\n        model_name=\"bert\",\n        root=\"\",\n    ):\n        self.tokens = tokens\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.model_name = model_name\n\n        self.article = load_text(id_, root=root)\n        self.texts = self.article_to_texts()\n\n    def __len__(self):\n        return len(self.texts)\n\n    def article_to_texts(self):\n        texts = []\n        for section in self.article:\n            sentences = remove_spaces(section[\"text\"]).split(\". \")\n\n            text = \"\"\n            for i in range(len(sentences)):\n                if (\n                    len(self.tokenizer.encode(text + sentences[i] + \". \")) < self.max_len - 2\n                ):\n                    text += sentences[i] + \". \"\n                else:\n                    texts.append(text)\n                    text = sentences[i] + \". \"\n\n            if len(text):\n                texts.append(text)\n\n        return texts\n\n    def article_to_texts_2(self):\n        texts = []\n        text = \"\"\n        for section in self.article:\n            sentences = remove_spaces(section[\"text\"]).split(\". \")\n\n            for i in range(len(sentences)):\n                if (\n                    len(self.tokenizer.encode(text + sentences[i] + \". \"))\n                    < self.max_len - 2\n                ):\n                    text += sentences[i] + \". \"\n                else:\n                    texts.append(text)\n                    text = sentences[i] + \". \"\n\n        if len(text):\n            texts.append(text)\n\n        return texts\n\n    def __getitem__(self, idx):\n#         print(self.texts[idx])\n        data = process_data_inf(\n            self.texts[idx],\n            self.tokenizer,\n            self.tokens,\n            max_len=self.max_len,\n            model_name=self.model_name,\n        )\n\n        return {\n            \"ids\": torch.tensor(data[\"ids\"], dtype=torch.long),\n            \"token_type_ids\": torch.tensor(data[\"token_type_ids\"], dtype=torch.long),\n            \"text\": data[\"text\"],\n            \"offsets\": torch.tensor(data[\"offsets\"], dtype=torch.float),\n        }\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.758595Z","iopub.execute_input":"2021-06-14T21:02:38.759083Z","iopub.status.idle":"2021-06-14T21:02:38.774815Z","shell.execute_reply.started":"2021-06-14T21:02:38.759053Z","shell.execute_reply":"2021-06-14T21:02:38.773698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"TRANSFORMERS = {\n    \"roberta-base\": (RobertaModel, \"roberta-base\", RobertaConfig),\n    \"albert-base-v2\": (AlbertModel, \"albert-base-v2\", AlbertConfig),\n    \"albert-large-v2\": (AlbertModel, \"albert-large-v2\", AlbertConfig),\n    \"albert-xlarge-v2\": (AlbertModel, \"albert-xlarge-v2\", AlbertConfig),\n    \"albert-xxlarge-v2\": (AlbertModel, \"albert-xxlarge-v2\", AlbertConfig),\n    \"bert-base-uncased\": (BertModel, \"bert-base-uncased\", BertConfig),\n    \"bert-base-cased\": (BertModel, \"bert-base-cased\", BertConfig),\n    \"bert-large-uncased-whole-word-masking\": (\n        BertModel,\n        \"bert-large-uncased-whole-word-masking\",\n        BertConfig,\n    ),\n    \"distilbert-base-uncased-distilled-squad\": (\n        DistilBertModel,\n        \"distilbert-base-uncased-distilled-squad\",\n        BertConfig,\n    ),\n}\n\n\nclass NERTransformer(nn.Module):\n    def __init__(\n        self,\n        model,\n        nb_layers=1,\n        nb_ft=None,\n        k=5,\n        drop_p=0.1,\n        multi_sample_dropout=False,\n        use_squad_weights=False,\n    ):\n        super().__init__()\n        self.name = model\n        self.nb_layers = nb_layers\n        self.multi_sample_dropout = multi_sample_dropout\n\n        self.pad_idx = 1 if \"roberta\" in self.name else 0\n\n        model_class, _, config_class = TRANSFORMERS[model]\n\n        try:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'bert_config.json')\n        except:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'config.json')\n        config.output_hidden_states = True\n\n        self.transformer =  model_class(config)\n\n        if \"distil\" in self.name:\n            self.nb_features = self.transformer.transformer.layer[\n                -1\n            ].ffn.lin2.out_features\n        elif \"albert\" in self.name:\n            self.nb_features = (\n                self.transformer.encoder.albert_layer_groups[-1]\n                .albert_layers[-1]\n                .ffn_output.out_features\n            )\n        else:\n            self.nb_features = self.transformer.pooler.dense.out_features\n\n        if nb_ft is None:\n            nb_ft = self.nb_features\n\n        self.logits = nn.Sequential(\n            # nn.Linear(nb_ft * 2, nb_ft),\n            # nn.Tanh(),\n            nn.Linear(nb_ft, 1),\n        )\n        self.cnn = nn.Sequential(\n            nn.Conv1d(self.nb_features * self.nb_layers, nb_ft * 2, kernel_size=k, padding=k // 2),\n            nn.Tanh(),\n            nn.Dropout(drop_p),\n            nn.Conv1d(nb_ft * 2, nb_ft, kernel_size=k, padding=k // 2),\n            nn.Tanh(),\n            nn.Dropout(drop_p),\n        )\n\n        self.high_dropout = nn.Dropout(p=0.5)\n\n    def forward(self, tokens, token_type_ids):\n        \"\"\"\n        Usual torch forward function\n\n        Arguments:\n            tokens {torch tensor} -- Sentence tokens\n            token_type_ids {torch tensor} -- Sentence tokens ids\n        \"\"\"\n\n        if \"distil\" in self.name:\n            hidden_states = self.transformer(\n                tokens,\n                attention_mask=(tokens != self.pad_idx).long(),\n            )[-1]\n        else:\n            hidden_states = self.transformer(\n                tokens,\n                attention_mask=(tokens != self.pad_idx).long(),\n                token_type_ids=token_type_ids,\n            )[-1]\n\n        hidden_states = hidden_states[::-1]\n        features = torch.cat(hidden_states[:self.nb_layers], -1)\n\n        if self.multi_sample_dropout and self.training:\n            logits = torch.mean(\n                torch.stack(\n                    [\n                        self.logits(\n                            self.cnn(self.high_dropout(features).transpose(1, 2)).transpose(1, 2)\n                        )\n                        for _ in range(5)\n                    ],\n                    dim=0,\n                ),\n                dim=0,\n            )\n        else:\n            logits = self.logits(\n                self.cnn(features.transpose(1, 2)).transpose(1, 2)\n            )\n\n        return logits\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.777239Z","iopub.execute_input":"2021-06-14T21:02:38.777638Z","iopub.status.idle":"2021-06-14T21:02:38.798751Z","shell.execute_reply.started":"2021-06-14T21:02:38.777600Z","shell.execute_reply":"2021-06-14T21:02:38.797779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\n","metadata":{}},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"def predict(model, dataset, batch_size=32, activation=\"sigmoid\"):\n    \"\"\"\n    Usual predict torch function\n    \"\"\"\n    model.eval()\n\n    loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=NUM_WORKERS,\n        pin_memory=True,\n    )\n\n    preds = []\n    with torch.no_grad():\n        for data in loader:\n            # ids, token_type_ids = trim_tensors(\n            #     [data[\"ids\"], data[\"token_type_ids\"]],\n            #     model.name\n            # )  # TODO : doesn't work with concat\n            ids, token_type_ids = data[\"ids\"], data[\"token_type_ids\"]\n\n            y_pred = model(ids.cuda(), token_type_ids.cuda())\n\n            if activation == \"sigmoid\":\n                y_pred = torch.sigmoid(y_pred)\n            elif activation == \"softmax\":\n                y_pred = torch.softmax(y_pred, 2)\n\n            preds.append(y_pred.detach().cpu().numpy())\n\n    return np.concatenate(preds)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.800199Z","iopub.execute_input":"2021-06-14T21:02:38.800693Z","iopub.status.idle":"2021-06-14T21:02:38.810865Z","shell.execute_reply.started":"2021-06-14T21:02:38.800658Z","shell.execute_reply":"2021-06-14T21:02:38.810041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicted strings from probas","metadata":{}},{"cell_type":"code","source":"def get_predicted_string(text, offset, pred, threshold=0.5):\n    text_preds, text_probs, starts = [], [], []\n    pred_str, proba, start = \"\", [], None\n    for j in range(1, len(pred) - 1):\n        if pred[j] > threshold:\n            pred_str += text[offset[j][0]: offset[j][1]]\n            if start is None:\n                start = offset[j][0]\n            proba.append(pred[j])\n            if j + 1 < len(offset) and offset[j][1] < offset[j + 1][0]:\n                pred_str += \" \"\n\n        elif pred[j] < threshold and len(pred_str):\n            text_preds.append(pred_str)\n            text_probs.append(proba)\n            starts.append(start)\n            pred_str, proba, start = \"\", [], None\n\n    if len(pred_str):\n        text_preds.append(pred_str)\n        text_probs.append(proba)\n\n    return text_preds, text_probs, starts\n\n\ndef get_pred_from_probas(dataset, probas, threshold=0.5, min_confidence=0.9):\n    predicted_strings = []\n    for i in range(len(dataset)):\n        data = dataset[i]\n        pred = probas[i]\n        offset = data[\"offsets\"].cpu().numpy().astype(int)\n        text = data[\"text\"]\n\n        text_preds, text_probs, starts = get_predicted_string(\n            text, offset, pred, threshold=0.5\n        )\n\n        preds_pp = []\n        for text_pred, text_prob, start in zip(text_preds, text_probs, starts):\n            new_pred = text_pred\n            \n            # not full word\n            if start > 0:\n                if text[start - 1] != \" \" and text[start] != \" \":\n                    new_pred = \" \".join(text_pred.split(\" \")[1:])\n\n            if start + len(text_pred) < len(text):\n                if (\n                    text[start + len(text_pred)] != \" \"\n                    and text[start + len(text_pred) - 1] != \" \"\n                ):\n                    new_pred = \" \".join(new_pred.split(\" \")[:-1])\n\n            # Way too short\n            if len(new_pred.split(\" \")) < 2:\n                continue\n            elif len(new_pred) < 5:\n                continue\n\n            # Low confidence\n            if np.max(text_prob) < min_confidence:\n                continue\n\n            preds_pp.append(new_pred.strip())\n\n        predicted_strings += preds_pp\n\n    return predicted_strings","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.812205Z","iopub.execute_input":"2021-06-14T21:02:38.812647Z","iopub.status.idle":"2021-06-14T21:02:38.829350Z","shell.execute_reply.started":"2021-06-14T21:02:38.812550Z","shell.execute_reply":"2021-06-14T21:02:38.828521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post-processing","metadata":{}},{"cell_type":"markdown","source":"### Overlap","metadata":{}},{"cell_type":"code","source":"nationalities = [clean_text(txt) for txt in pd.read_csv('../input/coleridge-resources/nationalities.csv')['Nationality'].values]\ncountries = [clean_text(txt) for txt in pd.read_csv('../input/coleridge-resources/countries.csv')['Name'].values]\nus_states = [clean_text(txt) for txt in pd.read_csv('../input/coleridge-resources/us_states.csv')['State'].values]\n\nus_states_abv = [clean_text(txt) for txt in pd.read_csv('../input/coleridge-resources/us_states.csv')['Abbreviation'].values]\nus_states_abv += [a[0] + ' ' + a[1] for a in us_states_abv]\n\nSUFFIXES = [\n    'data', 'dataset', 'data set', 'data sets', 'datasets', 'database', 'databases', 'catalog', 'catalogs',\n    'survey', 'surveys', 'study', 'studies', 'census', 'program', 'programs', 'assessment', 'assessments',\n    'registry', 'registries', 'list', 'lists', 'network', 'networks', 'archive', 'archives'\n]\n\nPREFIXES = nationalities + countries + us_states + us_states_abv + [\n    'international', 'national', 'european', 'europe', 'american', 'america', 'african', 'africa', 'asian', 'asia', \n    'u s', 'us', 'uk', 'u k']\n\n\ndef post_process_overlap(preds, text):\n#     preds = np.unique([clean_text(p) for p in preds])\n    \n    to_remove = []\n    for i in range(len(preds)):\n        for j in range(len(preds)):\n            if i != j:\n                if preds[i] in preds[j]:\n                    count_i = text.count(preds[i])\n                    count_j = text.count(preds[j])\n                    \n                    if count_i == count_j:  \n                        # i is included in j everywhere, we keep j\n                        to_keep = [j]\n                        to_remove.append(i)\n                    else:\n                        before = preds[j][:preds[j].find(preds[i])].strip()\n                        after = preds[j][preds[j].find(preds[i]) + len(preds[i]):].strip()\n                        \n                        if after in SUFFIXES and not len(before):\n                            pass\n                        \n                        elif before in PREFIXES and not len(after):\n                            pass\n                            \n                        elif after in SUFFIXES and before in PREFIXES:\n                            pass\n                            \n                        elif len(after.split(' ')) < 4 and after.split(' ')[-1] in SUFFIXES:\n                            to_remove.append(i)\n                            \n                        else:\n                            to_remove.append(j)\n                    \n    preds_pp = [p for i, p in enumerate(preds) if i not in to_remove]\n    return preds_pp","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.830995Z","iopub.execute_input":"2021-06-14T21:02:38.831365Z","iopub.status.idle":"2021-06-14T21:02:38.896423Z","shell.execute_reply.started":"2021-06-14T21:02:38.831328Z","shell.execute_reply":"2021-06-14T21:02:38.895724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fuzzywuzzy","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(DATA_PATH + 'train.csv')\noriginal_datasets = set(\n    list(df['dataset_title'].apply(clean_text)) +\n    list(df['dataset_label'].apply(clean_text)) +\n    list(df['cleaned_label'])\n)\n\n# extra_datasets = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')['title'].values.tolist()\nextra_datasets = pd.read_csv('../input/coleridge-resources/datasets_100k.csv')['title'].values.tolist()\nextra_datasets = list(dict.fromkeys([clean_text(d) for d in extra_datasets]))\nextra_datasets = set(extra_datasets)\n\ncandidates = pd.read_csv(\"../input/coleridge-resources/candidates_done_2.csv\", sep=\";\").dropna()\ncandidates = candidates[candidates['status'] == 'y']\ncandidates_datasets = candidates[candidates['status_sure'] == 'y']['label']\ncandidates_datasets = set([clean_text(d) for d in candidates_datasets])\n\nDATASETS  = list(original_datasets.union(extra_datasets).union(candidates_datasets))\n\nprint(f'Retrieved {len(DATASETS)} datasets')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:38.898092Z","iopub.execute_input":"2021-06-14T21:02:38.898326Z","iopub.status.idle":"2021-06-14T21:02:40.287967Z","shell.execute_reply.started":"2021-06-14T21:02:38.898303Z","shell.execute_reply":"2021-06-14T21:02:40.287053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fuzzywuzzy import fuzz\n\nIS_OK = []\nIS_NOT_OK = []\n\ndef post_process_fuzz(all_preds, datasets, threshold=60):\n    all_preds_pp = []\n\n    for preds in all_preds:\n        preds_pp = []\n        for pred in preds:\n            \n            if pred in IS_OK:\n                preds_pp.append(pred)\n                break\n                \n            elif pred in IS_NOT_OK:\n                break\n            \n            found = False\n            for dataset in datasets:\n                score = fuzz.ratio(dataset, pred)\n                if score > threshold:\n                    preds_pp.append(pred)\n                    IS_OK.append(pred)\n                    found = True\n                    break\n            \n            if not found:\n#                 print(f'removed {pred}')\n                IS_NOT_OK.append(pred)\n        all_preds_pp.append(preds_pp)\n    return all_preds_pp","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:40.289282Z","iopub.execute_input":"2021-06-14T21:02:40.289849Z","iopub.status.idle":"2021-06-14T21:02:40.312127Z","shell.execute_reply.started":"2021-06-14T21:02:40.289801Z","shell.execute_reply":"2021-06-14T21:02:40.311366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Others","metadata":{}},{"cell_type":"code","source":"KEYWORDS = [\n    'initiative',\n    \"data\",\n    \"dataset\",\n    \"database\",\n    \"catalog\",\n    \"survey\",\n    \"study\",\n    \"studies\",\n    \"census\",\n    \"program\",\n    'assessment',\n    \"registry\",\n    \"registries\",\n    \"list\",\n    \"index\",\n    'archive',\n    'inventory',\n    'inventories',\n    'network',\n    'model',\n    'system',\n    'report',\n]\n\ndef post_process_indicators(preds):\n    preds_pp = []\n    for pred in preds:\n        pred_pp = []\n        for p in pred:\n            if any([r in clean_text(p) for r in KEYWORDS]):\n                pred_pp.append(p)\n        preds_pp.append(pred_pp)\n    \n    return preds_pp\n\n\ndef post_process_len(preds, min_len=10):\n    new_preds = []\n    for pred in preds:\n        new_pred = [p for p in pred if len(p) > min_len]\n        new_preds.append(new_pred)\n    return new_preds","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:40.314922Z","iopub.execute_input":"2021-06-14T21:02:40.315164Z","iopub.status.idle":"2021-06-14T21:02:40.325590Z","shell.execute_reply.started":"2021-06-14T21:02:40.315140Z","shell.execute_reply":"2021-06-14T21:02:40.324709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Overall","metadata":{}},{"cell_type":"code","source":"def post_process(preds, text, preds_matching=[], min_len=10):\n    preds = [clean_text(pred) for pred in preds]\n    preds = post_process_len([preds])[0]\n    \n    preds = post_process_indicators([preds])[0]\n    \n    preds += preds_matching\n    preds = np.unique(preds)\n    \n    preds = post_process_fuzz([preds], DATASETS, threshold=90)[0]\n    \n    preds = post_process_overlap(preds, text)\n\n    \n    return '|'.join(preds)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:40.327037Z","iopub.execute_input":"2021-06-14T21:02:40.327746Z","iopub.status.idle":"2021-06-14T21:02:40.336787Z","shell.execute_reply.started":"2021-06-14T21:02:40.327706Z","shell.execute_reply":"2021-06-14T21:02:40.335801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## $k$-fold","metadata":{}},{"cell_type":"code","source":"def k_fold_inference(config, df, weights, root=\"\", min_confidence=0.5, min_len=5):\n    tokenizer, tokens = create_tokenizer_and_tokens(config)\n\n    models = []\n    for w in weights:\n        model = NERTransformer(\n            config.selected_model,\n            nb_layers=config.nb_layers,\n            nb_ft=config.nb_ft,\n            k=config.conv_kernel,\n            drop_p=config.drop_p,\n            multi_sample_dropout=config.multi_sample_dropout,\n        ).cuda()\n        model.zero_grad()\n        load_model_weights(model, w)\n        models.append(model)\n\n    original_datasets = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')['title'].values.tolist()\n    original_datasets = list(dict.fromkeys([clean_text(d) for d in original_datasets]))\n    original_datasets.remove(\"clinicaltrials gov\")\n\n    all_preds = []\n    for text_id in tqdm(df[\"Id\"]):\n\n        dataset = ArticleDataset(\n            text_id,\n            tokenizer,\n            tokens,\n            max_len=config.max_len,\n            model_name=config.selected_model,\n            root=root,\n        )\n        \n        text = ''\n        for section in dataset.article:\n            text += section['text'] + ' '\n        text = (clean_text(text))\n\n        probas = []\n        for model in models:\n            proba = predict(\n                model, dataset, batch_size=config.val_bs, activation=config.activation\n            )\n            probas.append(proba)\n        probas = np.mean(probas, 0)\n\n        preds = get_pred_from_probas(dataset, probas, min_confidence=min_confidence)\n        \n        preds_matching = []\n#         for dataset in original_datasets:\n#             if dataset in text:\n#                 preds_matching.append(dataset)\n                    \n        preds = post_process(preds, text, preds_matching, min_len=min_len)\n\n        all_preds.append(preds)\n\n    return all_preds","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:02:40.338226Z","iopub.execute_input":"2021-06-14T21:02:40.338675Z","iopub.status.idle":"2021-06-14T21:02:40.351174Z","shell.execute_reply.started":"2021-06-14T21:02:40.338640Z","shell.execute_reply":"2021-06-14T21:02:40.350166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"# EXP_FOLDER = \"../input/coleridge-cp/roberta_04-06_12/roberta_04-06_12/\"\n# EXP_FOLDER = \"../input/coleridge-cp/roberta_07-06_0/roberta_07-06_0/\"\n# EXP_FOLDER = \"../input/coleridge-cp/roberta_29-05_0/roberta_29-05_0/\"\nEXP_FOLDER = \"../input/coleridge-cp/roberta_09-06_2/roberta_09-06_2/\"\n# EXP_FOLDER = \"../input/coleridge-cp/roberta_12-06_2/roberta_12-06_2/\"","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.353079Z","iopub.execute_input":"2021-06-14T21:02:40.353477Z","iopub.status.idle":"2021-06-14T21:02:40.362393Z","shell.execute_reply.started":"2021-06-14T21:02:40.353425Z","shell.execute_reply":"2021-06-14T21:02:40.361751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = sorted(glob.glob(EXP_FOLDER + \"*.pt\"))\n\n# weights = weights[:1]\nprint(f\" -> Found {len(weights)} models\")","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.363576Z","iopub.execute_input":"2021-06-14T21:02:40.363954Z","iopub.status.idle":"2021-06-14T21:02:40.385682Z","shell.execute_reply.started":"2021-06-14T21:02:40.363914Z","shell.execute_reply":"2021-06-14T21:02:40.384912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config(json.load(open(EXP_FOLDER + \"config.json\", 'r')))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.387355Z","iopub.execute_input":"2021-06-14T21:02:40.387725Z","iopub.status.idle":"2021-06-14T21:02:40.397980Z","shell.execute_reply.started":"2021-06-14T21:02:40.387689Z","shell.execute_reply":"2021-06-14T21:02:40.397174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(DATA_PATH + \"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.401118Z","iopub.execute_input":"2021-06-14T21:02:40.401409Z","iopub.status.idle":"2021-06-14T21:02:40.413959Z","shell.execute_reply.started":"2021-06-14T21:02:40.401384Z","shell.execute_reply":"2021-06-14T21:02:40.413028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MIN_LEN = 10\nMIN_CONFIDENCE = 0.9","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.416818Z","iopub.execute_input":"2021-06-14T21:02:40.417049Z","iopub.status.idle":"2021-06-14T21:02:40.423002Z","shell.execute_reply.started":"2021-06-14T21:02:40.417026Z","shell.execute_reply":"2021-06-14T21:02:40.422212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = k_fold_inference(\n    config,\n    df,\n    weights,\n    root=DATA_PATH_TEST,\n    min_len=MIN_LEN,\n    min_confidence=MIN_CONFIDENCE,\n)\n\ndf['PredictionString'] = preds\ndf.to_csv('submission.csv', index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:02:40.424322Z","iopub.execute_input":"2021-06-14T21:02:40.424756Z","iopub.status.idle":"2021-06-14T21:03:30.557859Z","shell.execute_reply.started":"2021-06-14T21:02:40.424720Z","shell.execute_reply":"2021-06-14T21:03:30.556960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching","metadata":{}},{"cell_type":"code","source":"def read_json_pub(filename, data_path=\"\", output='text'):\n    json_path = os.path.join(data_path, (filename + '.json'))\n    contents = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            contents.append(data.get('text'))\n\n    return ' '.join(contents)\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T21:03:30.559750Z","iopub.execute_input":"2021-06-14T21:03:30.560144Z","iopub.status.idle":"2021-06-14T21:03:30.568768Z","shell.execute_reply.started":"2021-06-14T21:03:30.560103Z","shell.execute_reply":"2021-06-14T21:03:30.567884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')['title'].values\n# sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n# path = '../input/coleridgeinitiative-show-us-the-data/test'\n\n# submission = pd.DataFrame(columns=[\"Id\", \"PredictionString\"])\n\n# for idx, row in sample_sub.iterrows():\n#     to_append = [row['Id'], '']\n    \n#     text = read_json_pub(row['Id'], path)\n#     clean_string = text_cleaning(text)\n    \n#     for query_string in datasets:\n#         if query_string in clean_string:\n#             if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n#                 to_append[1] = to_append[1] + '|' + clean_text(query_string)\n#             if to_append[1] == '':\n#                 to_append[1] = clean_text(query_string)\n    \n#     # Complete with predictions\n#     if len(df['PredictionString'][idx]):\n#         for query_string in df['PredictionString'][idx].split('|'):\n#             already_found = False\n\n#             matched = to_append[1].split('|')\n#             for match in matched:\n#                 if match in query_string or query_string in match:\n\n#                     already_found = True\n\n#             if not already_found:\n#                 to_append[1] = to_append[1] + '|' + query_string\n        \n#     # Replace only if empty\n# #     if not len(to_append[1]): # if none found, use model pred\n# #         to_append[1] = df['PredictionString'][idx]\n\n#     submission.loc[idx] = to_append\n\n# submission.to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T21:03:30.570038Z","iopub.execute_input":"2021-06-14T21:03:30.570406Z","iopub.status.idle":"2021-06-14T21:03:30.579064Z","shell.execute_reply.started":"2021-06-14T21:03:30.570369Z","shell.execute_reply":"2021-06-14T21:03:30.578254Z"},"trusted":true},"execution_count":null,"outputs":[]}]}