{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setting","metadata":{"papermill":{"duration":0.031891,"end_time":"2021-06-07T02:45:30.774980","exception":false,"start_time":"2021-06-07T02:45:30.743089","status":"completed"},"tags":[]}},{"cell_type":"code","source":"COMPUTE_CV = False\nEDA_DEMO = True\nALL_BLENDED = False\nBASELINE_HELPING = False\nMATCH_ONLY = False\nMLM_ONLY = False\nKEN_MATCHING = True\nBS_CLEANING = False\nTHEO_MERGE = False\nSEED = 42","metadata":{"papermill":{"duration":0.043473,"end_time":"2021-06-07T02:45:30.850244","exception":false,"start_time":"2021-06-07T02:45:30.806771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:54:05.869298Z","iopub.execute_input":"2021-06-17T02:54:05.869667Z","iopub.status.idle":"2021-06-17T02:54:05.874708Z","shell.execute_reply.started":"2021-06-17T02:54:05.869587Z","shell.execute_reply":"2021-06-17T02:54:05.873507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.030707,"end_time":"2021-06-07T02:45:30.912019","exception":false,"start_time":"2021-06-07T02:45:30.881312","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\nfrom IPython.display import clear_output\nclear_output()","metadata":{"_kg_hide-output":true,"papermill":{"duration":92.128851,"end_time":"2021-06-07T02:47:03.071955","exception":false,"start_time":"2021-06-07T02:45:30.943104","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:55:48.179537Z","iopub.execute_input":"2021-06-17T02:55:48.179853Z","iopub.status.idle":"2021-06-17T02:57:15.088908Z","shell.execute_reply.started":"2021-06-17T02:55:48.179824Z","shell.execute_reply":"2021-06-17T02:57:15.088135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{"papermill":{"duration":0.030726,"end_time":"2021-06-07T02:47:03.133556","exception":false,"start_time":"2021-06-07T02:47:03.102830","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.autonotebook import tqdm\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\nimport warnings\nwarnings.filterwarnings(\"ignore\", 'This pattern has match groups')\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nif len(sample_submission) > 4: COMPUTE_CV = False\nif COMPUTE_CV: \n    print('this submission notebook will compute CV score but commit notebook will not')\nelse:\n    print('this submission notebook will only be used to submit result')","metadata":{"papermill":{"duration":8.830004,"end_time":"2021-06-07T02:47:11.994219","exception":false,"start_time":"2021-06-07T02:47:03.164215","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:15.092270Z","iopub.execute_input":"2021-06-17T02:57:15.092545Z","iopub.status.idle":"2021-06-17T02:57:23.193443Z","shell.execute_reply.started":"2021-06-17T02:57:15.092518Z","shell.execute_reply":"2021-06-17T02:57:23.192504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"papermill":{"duration":0.031829,"end_time":"2021-06-07T02:47:12.058760","exception":false,"start_time":"2021-06-07T02:47:12.026931","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)\n\nif COMPUTE_CV: \n    sample_submission = train\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n    test_files_path = paper_test_folder\nelse:\n    sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n    test_files_path = paper_test_folder\n    \nadnl_govt_labels_path = '../input/bigger-govt-dataset-list/data_set_800.csv'","metadata":{"papermill":{"duration":0.178934,"end_time":"2021-06-07T02:47:12.269919","exception":false,"start_time":"2021-06-07T02:47:12.090985","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.194725Z","iopub.execute_input":"2021-06-17T02:57:23.195091Z","iopub.status.idle":"2021-06-17T02:57:23.301753Z","shell.execute_reply.started":"2021-06-17T02:57:23.195055Z","shell.execute_reply":"2021-06-17T02:57:23.301013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(sample_submission['Id']):\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"papermill":{"duration":0.108361,"end_time":"2021-06-07T02:47:12.411904","exception":false,"start_time":"2021-06-07T02:47:12.303543","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.303170Z","iopub.execute_input":"2021-06-17T02:57:23.303638Z","iopub.status.idle":"2021-06-17T02:57:23.377852Z","shell.execute_reply.started":"2021-06-17T02:57:23.303595Z","shell.execute_reply":"2021-06-17T02:57:23.377196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"papermill":{"duration":0.096962,"end_time":"2021-06-07T02:54:48.460137","exception":false,"start_time":"2021-06-07T02:54:48.363175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.380690Z","iopub.execute_input":"2021-06-17T02:57:23.380949Z","iopub.status.idle":"2021-06-17T02:57:23.435405Z","shell.execute_reply.started":"2021-06-17T02:57:23.380924Z","shell.execute_reply":"2021-06-17T02:57:23.434532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Additional Govt Datasets","metadata":{"papermill":{"duration":0.034448,"end_time":"2021-06-07T02:54:48.529137","exception":false,"start_time":"2021-06-07T02:54:48.494689","status":"completed"},"tags":[]}},{"cell_type":"code","source":"adnl_govt_labels = pd.read_csv(adnl_govt_labels_path)\n\nfor l in adnl_govt_labels.title:\n    all_labels.add(l)\n    \nall_labels = set(all_labels)\nprint(f'No. different labels: {len(all_labels)}')","metadata":{"papermill":{"duration":0.057428,"end_time":"2021-06-07T02:54:48.621253","exception":false,"start_time":"2021-06-07T02:54:48.563825","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.436753Z","iopub.execute_input":"2021-06-17T02:57:23.437133Z","iopub.status.idle":"2021-06-17T02:57:23.460182Z","shell.execute_reply.started":"2021-06-17T02:57:23.437097Z","shell.execute_reply":"2021-06-17T02:57:23.459397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{"papermill":{"duration":0.035312,"end_time":"2021-06-07T02:54:48.692225","exception":false,"start_time":"2021-06-07T02:54:48.656913","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\nif not BS_CLEANING:\n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        text = re.sub(' +', ' ', text)\n        emoji_pattern = re.compile(\"[\"\n                                   u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                                   u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                                   u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                                   u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                                   \"]+\", flags=re.UNICODE)\n        text = emoji_pattern.sub(r'', text)\n        return text\nelse:\n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = ''.join([k for k in text if k not in string.punctuation])\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        # text = re.sub(\"/'+/g\", ' ', text)\n        return text\n\n\ndef read_json_pub(filename, train_data_path=train_files_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"papermill":{"duration":0.053467,"end_time":"2021-06-07T02:54:48.780879","exception":false,"start_time":"2021-06-07T02:54:48.727412","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.462842Z","iopub.execute_input":"2021-06-17T02:57:23.463120Z","iopub.status.idle":"2021-06-17T02:57:23.474347Z","shell.execute_reply.started":"2021-06-17T02:57:23.463096Z","shell.execute_reply":"2021-06-17T02:57:23.473478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Ken Matching","metadata":{"papermill":{"duration":0.034114,"end_time":"2021-06-07T02:54:48.929833","exception":false,"start_time":"2021-06-07T02:54:48.895719","status":"completed"},"tags":[]}},{"cell_type":"code","source":"literal_preds = []\n\nif KEN_MATCHING and not MLM_ONLY:\n    literal_preds = []\n    to_append = []\n    for index, row in tqdm(sample_submission.iterrows()):\n        to_append = [row['Id'],'']\n        large_string = str(read_json_pub(row['Id'], test_files_path))\n        clean_string = text_cleaning(large_string)\n        for index, row2 in adnl_govt_labels.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n                    to_append[1] = to_append[1] + '|' + clean_text(query_string)\n                if to_append[1] == '':\n                    to_append[1] = clean_text(query_string)\n        literal_preds.append(*to_append[1:])\n\nelif MLM_ONLY:\n    print('This kernel will only use MLM model to predict.')","metadata":{"papermill":{"duration":1.341886,"end_time":"2021-06-07T02:54:50.306338","exception":false,"start_time":"2021-06-07T02:54:48.964452","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:23.475597Z","iopub.execute_input":"2021-06-17T02:57:23.476154Z","iopub.status.idle":"2021-06-17T02:57:24.538327Z","shell.execute_reply.started":"2021-06-17T02:57:23.476118Z","shell.execute_reply":"2021-06-17T02:57:24.537432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{"papermill":{"duration":0.03538,"end_time":"2021-06-07T02:54:50.377479","exception":false,"start_time":"2021-06-07T02:54:50.342099","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{"papermill":{"duration":0.035148,"end_time":"2021-06-07T02:54:50.448350","exception":false,"start_time":"2021-06-07T02:54:50.413202","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MATCH_ONLY:\n    PRETRAINED_PATH = '../input/coleridge-bert-mlmv4/output-mlm/checkpoint-48000'\n    TOKENIZER_PATH = '../input/coleridge-bert-mlmv4/model_tokenizer'\n\n    MAX_LENGTH = 64\n    OVERLAP = 20\n\n    PREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\n    DATASET_SYMBOL = '$' # this symbol represents a dataset name\n    NONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"papermill":{"duration":0.044572,"end_time":"2021-06-07T02:54:50.528296","exception":false,"start_time":"2021-06-07T02:54:50.483724","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:24.539752Z","iopub.execute_input":"2021-06-17T02:57:24.540354Z","iopub.status.idle":"2021-06-17T02:57:24.545344Z","shell.execute_reply.started":"2021-06-17T02:57:24.540311Z","shell.execute_reply":"2021-06-17T02:57:24.544640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to MLM format","metadata":{"papermill":{"duration":0.035406,"end_time":"2021-06-07T02:54:50.600441","exception":false,"start_time":"2021-06-07T02:54:50.565035","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Load model and tokenizer","metadata":{"papermill":{"duration":0.038863,"end_time":"2021-06-07T02:54:50.675016","exception":false,"start_time":"2021-06-07T02:54:50.636153","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not MATCH_ONLY:\n    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\n    model = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n\n    mlm = pipeline(\n        'fill-mask', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n    )","metadata":{"papermill":{"duration":10.216184,"end_time":"2021-06-07T02:55:00.926810","exception":false,"start_time":"2021-06-07T02:54:50.710626","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:24.546973Z","iopub.execute_input":"2021-06-17T02:57:24.547363Z","iopub.status.idle":"2021-06-17T02:57:39.776440Z","shell.execute_reply.started":"2021-06-17T02:57:24.547328Z","shell.execute_reply":"2021-06-17T02:57:39.775626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{"papermill":{"duration":0.035766,"end_time":"2021-06-07T02:55:00.998532","exception":false,"start_time":"2021-06-07T02:55:00.962766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"papermill":{"duration":0.056003,"end_time":"2021-06-07T02:55:01.091164","exception":false,"start_time":"2021-06-07T02:55:01.035161","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-17T02:57:39.777727Z","iopub.execute_input":"2021-06-17T02:57:39.778070Z","iopub.status.idle":"2021-06-17T02:57:39.794334Z","shell.execute_reply.started":"2021-06-17T02:57:39.778035Z","shell.execute_reply":"2021-06-17T02:57:39.793428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not MATCH_ONLY:\n    mask = mlm.tokenizer.mask_token\n    all_test_data = []\n    \n    for paper_id in tqdm(sample_submission['Id']):\n        # load paper\n        paper = papers[paper_id]\n\n        # extract sentences\n        sentences = set([clean_paper_sentence(sentence) for section in paper \n                         for sentence in section['text'].split('.')\n                        ])\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 1] # only accept sentences with length > 1 chars\n        sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n\n        # mask\n        test_data = []\n        for sentence in sentences:\n            for phrase_start, phrase_end in find_mask_candidates(sentence):\n                dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n                test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n\n        all_test_data.append(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:08:22.087725Z","iopub.execute_input":"2021-06-17T03:08:22.088062Z","iopub.status.idle":"2021-06-17T03:08:22.189668Z","shell.execute_reply.started":"2021-06-17T03:08:22.088029Z","shell.execute_reply":"2021-06-17T03:08:22.188832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform","metadata":{"papermill":{"duration":0.035403,"end_time":"2021-06-07T02:55:01.162367","exception":false,"start_time":"2021-06-07T02:55:01.126964","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Predict","metadata":{"papermill":{"duration":0.036731,"end_time":"2021-06-07T02:55:01.430195","exception":false,"start_time":"2021-06-07T02:55:01.393464","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import transformers\nimport torch.nn as nn\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-06-17T02:58:40.149469Z","iopub.execute_input":"2021-06-17T02:58:40.149799Z","iopub.status.idle":"2021-06-17T02:58:40.153857Z","shell.execute_reply.started":"2021-06-17T02:58:40.149769Z","shell.execute_reply":"2021-06-17T02:58:40.152853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetFinder(nn.Module):\n    \n    def __init__(self,params):\n        super().__init__()\n        self.model = transformers.AutoModel.from_pretrained(\"../input/scibert-huggingface/coleridge-scibert-models/output\")\n        for param in self.model.parameters():\n            param.requires_grad=False\n        self.dr = torch.nn.Dropout(params['dropout'])\n        self.fc1 = torch.nn.Linear(768,params['lstm_inp_size'])\n        self.relu = torch.nn.ReLU(inplace=True)\n        self.lstm = torch.nn.LSTM(input_size=params['lstm_inp_size'],hidden_size=params['hid_size'],bidirectional=True,batch_first=True)\n        self.fc = torch.nn.Linear(2*params['hid_size'],1)\n        self.e=0\n    def forward(self,inp):\n        inp = self.dr(self.model(**inp).last_hidden_state)\n        inp = self.relu(self.fc1(inp))\n        inp,_=self.lstm(inp)\n        inp = self.fc(inp).squeeze(2)\n        return torch.sigmoid(inp)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T02:58:41.904267Z","iopub.execute_input":"2021-06-17T02:58:41.904596Z","iopub.status.idle":"2021-06-17T02:58:41.912180Z","shell.execute_reply.started":"2021-06-17T02:58:41.904566Z","shell.execute_reply":"2021-06-17T02:58:41.911310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.AutoTokenizer.from_pretrained('../input/scibert-huggingface/coleridge-scibert-models/output')\nDEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nM_COUNT=5\n\nparams = {\n        'lr':0.000150074,\n        'loss_func':torch.nn.BCELoss(),\n        'lstm_inp_size': 2**10,\n        'dropout': 0.0024323,\n        'hid_size':2**12\n}  \n\nmodels = [DatasetFinder(params) for i in range(M_COUNT)]\nfor i in range(M_COUNT):\n    models[i].load_state_dict(torch.load(f'../input/processed-train-data-sentence-segmentaion/model_best_{i}.state'))\n    for params in models[i].parameters():\n        params.requires_grad=False\n    models[i].eval()\n    models[i] = models[i].to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T02:58:53.394881Z","iopub.execute_input":"2021-06-17T02:58:53.395276Z","iopub.status.idle":"2021-06-17T03:00:24.401266Z","shell.execute_reply.started":"2021-06-17T02:58:53.395245Z","shell.execute_reply":"2021-06-17T03:00:24.400453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_dataset_ann(text):\n    t = text.split('.')\n    p = 0\n    inp_id = []\n    attention_mask = []\n    for x in t:\n        com = tokenizer(x,max_length=30,padding=\"max_length\",truncation=True)\n        inp_id.append(torch.tensor(com[\"input_ids\"],dtype=torch.long).view(1,-1))\n        attention_mask.append(torch.tensor(com[\"attention_mask\"],dtype=torch.long).view(1,-1))\n#     ids = np.random.choice(np.arange(len(inp_id)),BATCH_SIZE)\n#     if(len(inp_id)<=BATCH_SIZE):\n#         ids = np.arange(len(inp_id))\n    inp = {\"input_ids\":torch.cat(inp_id,0).to(DEVICE),\n                 \"attention_mask\":torch.cat(attention_mask,0).to(DEVICE)}\n    out = None\n    for model in models:\n        if out is None:\n            out = model(inp).detach().cpu()\n        else:\n            out = out + model(inp).detach().cpu().numpy()\n    out = out/M_COUNT\n    ans = (inp['input_ids'].detach().cpu()*(out>0.45)).numpy()\n    answers = []\n    for i in ans:\n        if(i.sum()>0):\n            tmp = []\n            for x in i:\n                if x==0:\n                    word = tokenizer.decode(tmp)\n                    if len(word.split())>2:\n                        answers.append(clean_text(word))\n                    tmp = []\n                else:\n                    tmp.append(x)\n            word = tokenizer.decode(tmp)\n            if len(word.split())>2:\n                answers.append(clean_text(word))\n    if len(answers)==0:\n        return \"\"\n#     c = []\n#     for ans in answers:\n#         c.append(text.count(ans))\n#     answers = [answers[i] for i in np.argsort(c)[::-1][:3]]\n    return \"|\".join(answers) ","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:00:24.402945Z","iopub.execute_input":"2021-06-17T03:00:24.403277Z","iopub.status.idle":"2021-06-17T03:00:24.416135Z","shell.execute_reply.started":"2021-06-17T03:00:24.403239Z","shell.execute_reply":"2021-06-17T03:00:24.414662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"../input/coleridgeinitiative-show-us-the-data/test\"","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:00:24.417383Z","iopub.execute_input":"2021-06-17T03:00:24.417788Z","iopub.status.idle":"2021-06-17T03:00:24.437667Z","shell.execute_reply.started":"2021-06-17T03:00:24.417751Z","shell.execute_reply":"2021-06-17T03:00:24.436770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dsets = []\nids = []\nunused = []\nfor i, row in tqdm(sample_submission.iterrows()):\n    if len(literal_preds[i])<=1:\n        unused.append(i)\n        text = json.load(open(os.path.join(test_path,sample_submission.iloc[i,0]+\".json\")))\n        sec = []\n        for x in text:\n            sec.append(x['section_title'])\n            sec.append(\" \")    \n            sec.append(x['text'])    \n        entire = \"\".join(sec)\n        try:\n            pred = find_dataset_ann(entire)\n            literal_preds[i] = pred\n            spl = pred.split(\"|\")\n            for s in spl:\n                dsets.append(s)\n                ids.append(i)\n        except:\n            torch.cuda.empty_cache()\n            print(\"error\")\ndatasets = pd.DataFrame()\ndatasets['ids']=ids\ndatasets['dsets'] = dsets","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:08:30.007644Z","iopub.execute_input":"2021-06-17T03:08:30.007955Z","iopub.status.idle":"2021-06-17T03:08:30.052280Z","shell.execute_reply.started":"2021-06-17T03:08:30.007924Z","shell.execute_reply":"2021-06-17T03:08:30.051285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in unused:\n    test_data = all_test_data[i]\n    pred_bag = set()\n\n    if len(test_data):\n        texts, phrases = list(zip(*test_data))\n        mlm_pred = []\n        for p_id in range(0, len(texts), PREDICT_BATCH):\n            batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n            batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n\n            if len(batch_texts) == 1:\n                batch_pred = [batch_pred]\n\n            mlm_pred.extend(batch_pred)\n\n        for (result1, result2), phrase in zip(mlm_pred, phrases):\n            if (result1['score'] > result2['score']*2 and result1['token_str'] == DATASET_SYMBOL) or\\\n               (result2['score'] > result1['score']*2 and result2['token_str'] == NONDATA_SYMBOL):\n                pred_bag.add(clean_text(phrase))\n\n    # filter labels by jaccard score \n    filtered_labels = []\n\n    for label in sorted(pred_bag, key=len, reverse=True):\n        if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n            filtered_labels.append(label)\n    for s in filtered_labels:\n        dsets.append(s)\n        ids.append(i)\n    if len(literal_preds[i])>3:\n        literal_preds[i] = literal_preds[i]+'|'+ '|'.join(filtered_labels)\n    else:\n        literal_preds[i] = '|'.join(filtered_labels)","metadata":{"papermill":{"duration":23.160434,"end_time":"2021-06-07T02:55:24.628034","exception":false,"start_time":"2021-06-07T02:55:01.467600","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2021-06-17T03:12:07.567989Z","iopub.execute_input":"2021-06-17T03:12:07.568306Z","iopub.status.idle":"2021-06-17T03:12:07.578745Z","shell.execute_reply.started":"2021-06-17T03:12:07.568276Z","shell.execute_reply":"2021-06-17T03:12:07.577756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in set(ids):\n    ans = literal_preds[i]\n    spl = ans.split(\"|\")\n    n_spl = []\n    for s in spl:\n        if len(datasets[(datasets.ids!=i)&(datasets.dsets==s)])>0:\n            n_spl.append(s)\n    literal_preds[i] = \"|\".join(set(n_spl))","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:13:51.498147Z","iopub.execute_input":"2021-06-17T03:13:51.498480Z","iopub.status.idle":"2021-06-17T03:13:51.505462Z","shell.execute_reply.started":"2021-06-17T03:13:51.498449Z","shell.execute_reply":"2021-06-17T03:13:51.504426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = literal_preds\nsample_submission[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\n\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:14:29.877591Z","iopub.execute_input":"2021-06-17T03:14:29.877916Z","iopub.status.idle":"2021-06-17T03:14:29.895598Z","shell.execute_reply.started":"2021-06-17T03:14:29.877886Z","shell.execute_reply":"2021-06-17T03:14:29.894607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:15:12.677765Z","iopub.execute_input":"2021-06-17T03:15:12.678145Z","iopub.status.idle":"2021-06-17T03:15:12.687655Z","shell.execute_reply.started":"2021-06-17T03:15:12.678111Z","shell.execute_reply":"2021-06-17T03:15:12.686815Z"},"trusted":true},"execution_count":null,"outputs":[]}]}