{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip uninstall --yes datasets\n!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip uninstall --yes fsspec\n!pip install ../input/ffspecsforchallenge2/fsspec-0.9.0-py3-none-any.whl\n#!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:49:30.294084Z","iopub.execute_input":"2021-06-14T08:49:30.294409Z","iopub.status.idle":"2021-06-14T08:50:06.061843Z","shell.execute_reply.started":"2021-06-14T08:49:30.294336Z","shell.execute_reply":"2021-06-14T08:50:06.060775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np \nimport pandas as pd \nimport json\nimport re \nfrom tqdm import tqdm\nimport tensorflow as tf\nimport spacy\nimport nltk\nimport torch\nimport os\n\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping\nfrom keras.models import Model\nfrom keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Input, GlobalAveragePooling1D\n\nfrom collections import Counter\nimport transformers\n\nimport keras\ntqdm.pandas()\nfrom keras.preprocessing.sequence import pad_sequences\n\nimport argparse\nimport math\nimport random\nimport logging\nimport os\nimport sys\nfrom dataclasses import dataclass, field\nfrom typing import Optional\n\nimport numpy as np\n\nfrom datasets import ClassLabel, load_dataset, load_metric\n\nimport transformers\nfrom transformers import (\n    AutoConfig,\n    AutoModelForTokenClassification,\n    AutoTokenizer,\n    DataCollatorForTokenClassification,\n    HfArgumentParser,\n    PreTrainedTokenizerFast,\n    Trainer,\n    TrainingArguments,\n    set_seed,\n    BertTokenizer\n)\nfrom transformers.trainer_utils import get_last_checkpoint, is_main_process\nfrom transformers.utils import check_min_version","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-06-14T08:50:06.063523Z","iopub.execute_input":"2021-06-14T08:50:06.063887Z","iopub.status.idle":"2021-06-14T08:50:16.124439Z","shell.execute_reply.started":"2021-06-14T08:50:06.063858Z","shell.execute_reply":"2021-06-14T08:50:16.123588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION = False #True\nTHRESHOLD = 0.75 #0.5\nBATCH = 1536\nprint(BATCH)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:16.126462Z","iopub.execute_input":"2021-06-14T08:50:16.126857Z","iopub.status.idle":"2021-06-14T08:50:16.134923Z","shell.execute_reply.started":"2021-06-14T08:50:16.126818Z","shell.execute_reply":"2021-06-14T08:50:16.133843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load all the data","metadata":{}},{"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\neval_df = pd.read_pickle(\"../input/give-us-the-data-in-sentences/eval_df_annotated.pkl\")\ntrain_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ndf = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:16.136559Z","iopub.execute_input":"2021-06-14T08:50:16.136924Z","iopub.status.idle":"2021-06-14T08:50:31.721550Z","shell.execute_reply.started":"2021-06-14T08:50:16.136889Z","shell.execute_reply":"2021-06-14T08:50:31.720489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = eval_df.drop(columns=['section_match', \"sentence_text\", \"text\"])\nall_sets = train_df.dataset_title.unique()\neval_df = eval_df[~eval_df.label.isin(all_sets)]\neval_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:31.723006Z","iopub.execute_input":"2021-06-14T08:50:31.723466Z","iopub.status.idle":"2021-06-14T08:50:32.843475Z","shell.execute_reply.started":"2021-06-14T08:50:31.723428Z","shell.execute_reply":"2021-06-14T08:50:32.842654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A small evaluation dataframe for testing generalization","metadata":{}},{"cell_type":"code","source":"eval_df = pd.concat([eval_df.loc[eval_df.sentence_match], eval_df.loc[~eval_df.sentence_match].sample(500)])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:32.844687Z","iopub.execute_input":"2021-06-14T08:50:32.845026Z","iopub.status.idle":"2021-06-14T08:50:33.103532Z","shell.execute_reply.started":"2021-06-14T08:50:32.844991Z","shell.execute_reply":"2021-06-14T08:50:33.102667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helping function for the processing of the data","metadata":{}},{"cell_type":"code","source":"import nltk.data\n\ntokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n\ndef get_text(sample_id, location='/kaggle/input/coleridgeinitiative-show-us-the-data/test/'):\n    \"\"\"\n    load the text for the given ID as string\n    \"\"\"\n    with open(location + sample_id + \".json\", \"r\") as file:\n        sample = json.loads(file.read())\n\n    return \" \".join([s['section_title'] + \" \"+ s['text'] for s in sample])\n\ndef read_json_from_folder(folder_name):\n    \"\"\"\n    load json object for the publications\n    \"\"\"\n    \n    json_dict = {}\n    for filename in os.listdir(folder_name):\n        with open(os.path.join(folder_name, filename)) as f:\n            json_dict[filename[:-5]] = json.load(f)\n    return json_dict\n\ndef get_section_data(sample_id,location='/kaggle/input/coleridgeinitiative-show-us-the-data/test/'): #train\n    \"\"\"\n    extract the sections for each paper\n    \"\"\"\n    with open(location + sample_id + \".json\", \"r\") as file:\n        raw_text = json.loads(file.read())\n        \n    return [(sample_id, s['section_title'], s['text']) for s in raw_text]\n\ndef clean_text_kaggle(txt):\n    \"\"\"\n    the original Kaggle cleaning function\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef custom_cleaning(text):\n    \"\"\"\n    our custom cleaning function\n    \"\"\"\n    # lower case text\n    text = str(text).lower()\n    # remove https and replace by hyperlink token -> The hyperlink itself might be a feature, by replacing it with a token we can utilize it without matching with the link text   \n    text = re.sub(r'(\\(?https?:[^\\s]*|\\(?www.[^\\s]*)', \"<HYPER>\",  text)\n    text = re.sub(f'\\\\n', \" \", text)\n    return text\n\ndef split_into_sentences(text):\n    \"\"\"\n    tokenize with the nltk tokenizer\n    \"\"\"\n    return tokenizer.tokenize(text)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.104921Z","iopub.execute_input":"2021-06-14T08:50:33.105259Z","iopub.status.idle":"2021-06-14T08:50:33.128238Z","shell.execute_reply.started":"2021-06-14T08:50:33.105224Z","shell.execute_reply":"2021-06-14T08:50:33.127361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Boyer Moore Matching algorithm\nRetrieved from https://gist.github.com/dbrgn/1154006","metadata":{}},{"cell_type":"code","source":"class last_occurrence(object):\n    \"\"\"Last occurrence functor.\"\"\"\n\n    def __init__(self, pattern, alphabet):\n        \"\"\"Generate a dictionary with the last occurrence of each alphabet\n        letter inside the pattern.\n        \n        Note: This function uses str.rfind, which already is a pattern\n        matching algorithm. There are more 'basic' ways to generate this\n        dictionary.\"\"\"\n        self.occurrences = dict()\n        for letter in alphabet:\n            self.occurrences[letter] = pattern.rfind(letter)\n\n    def __call__(self, letter):\n        \"\"\"Return last position of the specified letter inside the pattern.\n        Return -1 if letter not found in pattern.\"\"\"\n        return self.occurrences[letter]\n\n\ndef boyer_moore_match(text, pattern):\n    \"\"\"Find occurrence of pattern in text.\"\"\"\n    alphabet = set(text)\n    last = last_occurrence(pattern, alphabet)\n    m = len(pattern)\n    n = len(text)\n    i = m - 1  # text index\n    j = m - 1  # pattern index\n    while i < n:\n        if text[i] == pattern[j]:\n            if j == 0:\n                return i\n            else:\n                i -= 1\n                j -= 1\n        else:\n            l = last(text[i])\n            i = i + m - min(j, 1+l)\n            j = m - 1 \n    return -1","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.131996Z","iopub.execute_input":"2021-06-14T08:50:33.132399Z","iopub.status.idle":"2021-06-14T08:50:33.141530Z","shell.execute_reply.started":"2021-06-14T08:50:33.132359Z","shell.execute_reply":"2021-06-14T08:50:33.140687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pipeline","metadata":{}},{"cell_type":"markdown","source":"### Preprocessing\n\nPreprocessing the submission file for the pipeline","metadata":{}},{"cell_type":"code","source":"class Submission_Prep:\n    def __init__(self, csv = \"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\", input_folder = \"../input/coleridgeinitiative-show-us-the-data/test/\"):\n        \n        self.df = self.load_data(csv, input_folder)\n        \n    def load_data(self,csv, input_folder):\n        df = pd.read_csv(csv)\n        texts = df.Id.apply(lambda x: get_section_data(x,input_folder)).to_list()\n        df = pd.DataFrame([section for pub in texts for section in pub], columns=['pub_id', 'section', 'text'])\n        \n        sentences = df.text.apply(split_into_sentences)\n        df = df.assign(sentence_text = sentences)\n        df = df.drop(columns= [\"text\"])\n        \n        # split the text into individual sentences and add as separate rows\n        df =  df.explode('sentence_text')\n        df = df.assign(clean_text = df.sentence_text.apply(custom_cleaning))\n        return df","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.143513Z","iopub.execute_input":"2021-06-14T08:50:33.144031Z","iopub.status.idle":"2021-06-14T08:50:33.155499Z","shell.execute_reply.started":"2021-06-14T08:50:33.143993Z","shell.execute_reply":"2021-06-14T08:50:33.154512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = Submission_Prep().df\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.157094Z","iopub.execute_input":"2021-06-14T08:50:33.157476Z","iopub.status.idle":"2021-06-14T08:50:33.369662Z","shell.execute_reply.started":"2021-06-14T08:50:33.157439Z","shell.execute_reply":"2021-06-14T08:50:33.368842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_INPUT_SAVE_PATH = \"predict\"\nTEST_NER_DATA_FILE = \"test_ner.json\"\nPREDICT_BATCH = 64000\n\nclass BertPredictor:\n    \n    def __init__(self,model_path:str):\n        self.config = AutoConfig.from_pretrained(model_path,\n                num_labels=3,\n                finetuning_task='ner',\n                cache_dir=None,\n                revision=None,\n                use_auth_token=None,\n            )\n        self.tokenizer = AutoTokenizer.from_pretrained(model_path,\n                cache_dir=None,\n                use_fast=True,\n                revision=None,\n                use_auth_token=None,\n            )\n        self.model = AutoModelForTokenClassification.from_pretrained(model_path,\n                from_tf=False,\n                config=self.config,\n                cache_dir=False,\n                revision=False,\n                use_auth_token=None,\n            )\n        \n        # create the label list with our specific entities\n        self.label_list = ['B', 'I', 'O']\n        self.label_to_id = {l: i for i, l in enumerate(self.label_list)}\n        \n        # make a folder for the predictions\n        if not os.path.isdir(\"predict\"):\n            os.mkdir(\"predict\")\n        \n    def predict(self, data: pd.DataFrame):\n        \"\"\"\n        predicts and extracts the Datasets in the given data\n        \"\"\"\n        self.write_test_data(data)\n        predictions = self.get_prediction(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}')\n        processed_predictions = self.tranlate_output(predictions, data)\n        return processed_predictions\n    \n    def write_test_data(self, data: pd.DataFrame):\n        \"\"\"\n        the prediction code expects the input as json files.\n        Write one file and add dummy predictors\n        \"\"\"\n        test_rows = [] # test data in NER format\n        paper_length = [] # store the number of sentences each paper has\n        sentences = data.clean_text.to_list()\n        \n        # collect all sentences in json\n        for sentence in sentences:\n            sentence_words = sentence.split()\n            dummy_tags = ['O']*len(sentence_words)\n            test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n        \n        # track which sentence belongs to which data point\n        paper_length.append(len(sentences))\n        \n        for batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n            # write data rows to input file\n            with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w+') as f:\n                for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n                    json.dump(row, f)\n                    f.write('\\n')\n\n    \n    def load_test_dataset(self,test_file:str):\n        \"\"\"\n        load a Dataset from the json file\n        \"\"\"\n        dataset =  load_dataset(\"json\", data_files={\"test\": test_file})\n        return dataset[\"test\"]\n    \n    \n    def tokenize_and_align_labels(self, examples):\n        \n        tokenized_inputs = self.tokenizer(\n            examples[self.text_column_name],\n            max_length=128,\n            #padding=self.padding,\n            truncation=True,\n            # We use this argument because the texts in our dataset are lists of words (with a label for each word).\n            is_split_into_words=True,\n            )\n        labels = []\n        for i, label in enumerate(examples[self.label_column_name]):\n            word_ids = tokenized_inputs.word_ids(batch_index=i)\n            previous_word_idx = None\n            label_ids = []\n            for word_idx in word_ids:\n                # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n                # ignored in the loss function.\n                if word_idx is None:\n                    label_ids.append(-100)\n                # We set the label for the first token of each word.\n                elif word_idx != previous_word_idx:\n                    label_ids.append(self.label_to_id[label[word_idx]])\n                # For the other tokens in a word, we set the label to either the current label or -100, depending on\n                # the label_all_tokens flag.\n                else:\n                    label_ids.append(self.label_to_id[label[word_idx]])\n                previous_word_idx = word_idx\n\n            labels.append(label_ids)\n        tokenized_inputs[\"labels\"] = labels\n        return tokenized_inputs\n    \n   \n    def get_prediction(self,testfile:str): \n        dataset = self.load_test_dataset(testfile)\n        self.column_names = dataset.column_names\n        self.label_column_name = (\n            \"ner\" if \"ner\" in self.column_names else self.column_names[1]\n            )\n        \n        self.padding = \"max_length\"\n        self.text_column_name = \"tokens\" if \"tokens\" in self.column_names else self.column_names[0]\n        test_dataset = dataset.map(\n                    self.tokenize_and_align_labels,\n                    batched=True\n                )\n        \n        data_collator = DataCollatorForTokenClassification(self.tokenizer)\n        trainer = Trainer(\n                model=self.model,\n                tokenizer=self.tokenizer,\n                data_collator=data_collator,\n            )\n        \n        predictions, labels, metrics = trainer.predict(test_dataset)\n        predictions = np.argmax(predictions, axis=2)\n        \n        # Remove ignored index (special tokens)\n        true_predictions = [\n            [self.label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n            for prediction, label in zip(predictions, labels)\n        ]\n        return true_predictions\n    \n    def tranlate_output(self, results, submission):\n        \"\"\"\n        transform the output of the model back into strings. \n        \"\"\"\n        predicted_results = []\n        for sentence, labels in zip(submission.clean_text, results):\n            tokenized = self.tokenizer.tokenize(sentence)\n\n            prediction = []\n            for index, (word, label) in enumerate(zip(tokenized, labels)):\n                if label == \"B\":\n                    dataset = []\n                    dataset.append(word)\n                    next_id = index+1\n                    while len(labels) > next_id and labels[next_id] == \"I\":\n                        dataset.append(tokenized[next_id])\n                        next_id += 1\n                    prediction.append(dataset)\n\n            # in case we get several predictions for one sample, append them\n            if len(prediction) > 0:\n                transformed_prediction = [self.tokenizer.convert_tokens_to_string(dataset) for dataset in prediction]\n                predicted_results.append(\"|\".join(transformed_prediction))\n\n            # if no prediction, add empty string  \n            else:\n                predicted_results.append(\"\")        \n        return predicted_results\n            ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.371148Z","iopub.execute_input":"2021-06-14T08:50:33.371505Z","iopub.status.idle":"2021-06-14T08:50:33.398641Z","shell.execute_reply.started":"2021-06-14T08:50:33.371467Z","shell.execute_reply":"2021-06-14T08:50:33.397485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n    \n    def __init__(self, lstm_path, bert_path, threshold = THRESHOLD, batch = BATCH, gpu = False):\n    \n        self.lstm_model = self.load_lstm(lstm_path)\n\n        self.threshold = threshold\n        self.processing_batch = batch\n        \n        self.bert = BertPredictor(bert_path)\n        \n        self.dictionary = None\n        \n    def predict(self,df):\n        \n        df = df.reset_index()\n        \n        print(\"Start String matching\")\n        # check for  direct string matches\n        string_matched_df = self.string_matching(df)\n        print(\"String matching done\")\n        \n        # and split df based on the results\n        string_match = string_matched_df.dropna(subset=[\"PredictionString\"])\n        string_match = string_match.assign(lstm_prediction = pd.NA)\n        \n        no_string_match = string_matched_df[~string_matched_df.index.isin(string_match.index)]\n\n        # get lstm prediction\n        print(\"Start with the LSTM predictions\")\n        lstm_predictions = self.lstm_prediction(no_string_match.clean_text.to_list())\n        prediction_df = no_string_match.assign(lstm_prediction = lstm_predictions[:,1])\n        print(\"LSTM predictions done\")\n        \n        # filter out all samples above the threshold\n        might_have_ds = prediction_df[prediction_df.lstm_prediction > self.threshold]\n        no_ds = prediction_df[~prediction_df.index.isin(might_have_ds)]\n        no_ds = no_ds.assign(PredictionString = \"\")\n               \n        # get Scibert extraction\n        print(\"Start with the (Sci)bert extraction\")\n        # for all sentences which do not have direct string matches, use BERT\n        dataset_string = self.extract_dataset(might_have_ds)\n        print(\"(Sci)bert extraction done\")\n        # join both dfs together\n        final_df = pd.concat([string_match,no_ds, dataset_string])\n        return final_df\n        \n    def extract_dataset(self,df):\n        if len(df) > self.processing_batch * 2:\n            batches = np.split(df, int(len(df))/self.processing_batch)\n        else:\n            batches = np.split(df, 2)\n\n        success = 0\n        fail = 0\n        result_dfs = []\n        \n        for batch_df in batches:\n            try:\n                extracted_datasets = self.bert.predict(batch_df)    \n                result_dfs.append(batch_df.assign(PredictionString = extracted_datasets))\n                success +=1\n            except:\n                fail += 1\n                result_dfs.append(batch_df.assign(PredictionString = ['failed when retrieving bert']*len(batch_df)))   \n            print(f\"Bert extraction succeeded in {success} of the cases and failed in {fail}\")\n        return pd.concat(result_dfs)\n        \n    def add_dictionary(self,list_of_dataset):\n        \"\"\"\n        create the dictionary which contains all datasets from the training data\n        \"\"\"\n        clean_ds = set([custom_cleaning(ds) for ds in list_of_dataset if not ds == \"\"])\n        self.dictionary = list(clean_ds)\n\n    def lstm_prediction(self, text):\n        \"\"\"\n        obtain the predictions from the lstm model\n        \"\"\"\n        padded_x, tokenizer = self.tokenize(text)\n        tokens = tokenizer.texts_to_sequences(text)\n        predictions = self.lstm_model.predict(padded_x, batch_size = 32)\n        return predictions\n        \n    def load_lstm(self,path):\n        \"\"\"\n        loads the LSTM model into the pipeline\n        \"\"\"\n        model = keras.models.load_model(path)\n        return model\n        \n    \n    def tokenize(self, text, n_words=13615, sequence_length=140): \n        \"\"\"\n        tokenizes the text for the lstm model\n        \"\"\"\n        tokenizer = Tokenizer(num_words=n_words, filters=r'!\"#$%&()*+,-.:;<=>?@[\\]^_`{|}~', lower=True)\n        tokenizer.fit_on_texts(text)   \n        X = tokenizer.texts_to_sequences(text)\n        X = pad_sequences(X, maxlen=sequence_length, padding='pre')\n        return X, tokenizer\n    \n    def bert_prediction(self, df):\n        pass\n    \n    def string_matching(self, df):\n        results = df.clean_text.apply(lambda x: find_match(x, self.dictionary))\n        df = df.assign(PredictionString=results)\n        return df\n        \ndef find_match(sentence:str, dictionary:list):\n    \"\"\"\n    Literal string match\n    \"\"\"\n    try:\n        matches = [ds for ds in dictionary if boyer_moore_match(sentence, ds) != -1]\n        if len(matches) > 0:\n            # return the longest\n            return sorted(matches, key=len, reverse=True)[0]\n        else:\n            return pd.NA\n    except: \n        return f\"failed in Find match for sample: {sentence}\"\n\n    \npipeline = Pipeline(\"../input/lstm-for-ds/\", \"../input/coleridge-bert-models/output\", THRESHOLD, BATCH, gpu=True)\npipeline.add_dictionary([custom_cleaning(label) for label in train_df.dataset_label.unique()])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:50:33.400125Z","iopub.execute_input":"2021-06-14T08:50:33.400667Z","iopub.status.idle":"2021-06-14T08:50:49.784126Z","shell.execute_reply.started":"2021-06-14T08:50:33.400626Z","shell.execute_reply":"2021-06-14T08:50:49.783187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Execution","metadata":{}},{"cell_type":"code","source":"if SUBMISSION:\n    result = pipeline.predict(submission)\nelse:\n    result = pipeline.predict(eval_df)\n    \nresult.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-06-14T08:50:49.785478Z","iopub.execute_input":"2021-06-14T08:50:49.785838Z","iopub.status.idle":"2021-06-14T08:52:05.971408Z","shell.execute_reply.started":"2021-06-14T08:50:49.785803Z","shell.execute_reply":"2021-06-14T08:52:05.970604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Print a list of all predicted datasets","metadata":{}},{"cell_type":"code","source":"result.PredictionString.unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:05.972781Z","iopub.execute_input":"2021-06-14T08:52:05.973127Z","iopub.status.idle":"2021-06-14T08:52:05.980177Z","shell.execute_reply.started":"2021-06-14T08:52:05.973091Z","shell.execute_reply":"2021-06-14T08:52:05.979356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(result.PredictionString.unique())","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:05.981518Z","iopub.execute_input":"2021-06-14T08:52:05.982084Z","iopub.status.idle":"2021-06-14T08:52:05.996094Z","shell.execute_reply.started":"2021-06-14T08:52:05.982046Z","shell.execute_reply":"2021-06-14T08:52:05.995067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_list = []\npub_id_list = []\n\nfor pub_id, df in result.groupby(\"pub_id\"):\n    predictions = df.PredictionString.unique()\n    predictions = [p for p in predictions if p != \"\" and not isinstance(p, float)]\n    predicted_datasets = \"|\".join(predictions) if len(predictions) > 0 else \" \"\n    predictions_list.append(predicted_datasets)\n    pub_id_list.append(pub_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:05.997806Z","iopub.execute_input":"2021-06-14T08:52:05.998243Z","iopub.status.idle":"2021-06-14T08:52:06.248892Z","shell.execute_reply.started":"2021-06-14T08:52:05.998207Z","shell.execute_reply":"2021-06-14T08:52:06.248029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = pub_id_list\nsubmission['PredictionString'] = predictions_list\n\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:06.250296Z","iopub.execute_input":"2021-06-14T08:52:06.250809Z","iopub.status.idle":"2021-06-14T08:52:06.272739Z","shell.execute_reply.started":"2021-06-14T08:52:06.250773Z","shell.execute_reply":"2021-06-14T08:52:06.271747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_pickle(\"result.pkl\")","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:06.274082Z","iopub.execute_input":"2021-06-14T08:52:06.274448Z","iopub.status.idle":"2021-06-14T08:52:06.294873Z","shell.execute_reply.started":"2021-06-14T08:52:06.274412Z","shell.execute_reply":"2021-06-14T08:52:06.294086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r predict","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:06.296093Z","iopub.execute_input":"2021-06-14T08:52:06.296442Z","iopub.status.idle":"2021-06-14T08:52:06.995695Z","shell.execute_reply.started":"2021-06-14T08:52:06.296407Z","shell.execute_reply":"2021-06-14T08:52:06.994607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"code","source":"class Score:\n    def __init__(self, output, gt_col, pred_col):\n        \n        self.final_score = self.score_df_coleridge_initiative(output, gt_col, pred_col, beta=0.5, verbose=True) \n        #lambda output,gt_col,pred_col : self.score_df_coleridge_initiative(output, gt_col, pred_col, beta=0.5, verbose=True)  \n    \n    def score_df_coleridge_initiative(self,output, gt_col, pred_col, beta=0.5, verbose=True):\n        output['evaluation'] = output.apply(lambda x: self.coleridge_initiative_jaccard(x[gt_col], x[pred_col], verbose=False), axis=1)\n        output['js_scores'] = output['evaluation'].apply(lambda x : x[0])\n        output['pred_type'] = output['evaluation'].apply(lambda x : x[1])\n        output['tp_fp_fn'] = output['pred_type'].apply(lambda x : self.get_count_tp_fp_fn(x, verbose=False))\n        output = self.make_col_tp_fp_fn(output, 'tp_fp_fn')\n        tp = sum(output['TP'])\n        fp = sum(output['FP'])\n        fn = sum(output['FN'])\n        precision, recall = self.get_precision_recall(tp, fp, fn)\n        fbeta = self.fbeta_score(precision, recall, 0.5)\n        if verbose:\n            print(\"True Positives (TP) : \", tp)\n            print(\"False Positives (FP) : \", fp)\n            print(\"False Negatives (FN) : \", fn)\n            print(\"Precision : \", precision)\n            print(\"Recall : \", recall)\n            print(\"FBeta Score : \", fbeta)\n            display(output.head())\n        return fbeta\n    \n    def jaccard(self,str1, str2): \n        a = set(str1.lower().split()) \n        b = set(str2.lower().split())\n        c = a.intersection(b)\n        \n        # check for 0 in division here\n        if (len(a) + len(b) - len(c)) > 0:\n            return float(len(c)) / (len(a) + len(b) - len(c))\n        else:\n            return 0\n\n    def get_precision_recall(self,tp, fp, fn):\n        precision = tp / (tp+fp)\n        recall = tp / (tp + fn)\n        return precision, recall\n\n    def fbeta_score(self,precision, recall, beta):\n        fbeta = (1+(beta*beta))*((precision*recall)/( (beta*beta*precision) + recall))\n        return fbeta\n\n    def coleridge_initiative_jaccard(self,ground_truth, prediction, verbose=True):\n        gts = ground_truth.split('|')\n        pds = sorted(prediction.split('|'))\n        if verbose:\n            print(\"Ground truth : \" , gts)\n            print(\"Prediction : \", pds)\n        js_scores = []\n        cf_matrix = []\n        for pd in pds:\n            score = -1\n            for gt in gts:\n                js = self.jaccard(pd, gt)\n                if js > score:\n                    score = js\n            if score >= 0.5:\n                js_scores.append(score)\n                cf_matrix.append(\"TP\")\n            else:\n                js_scores.append(score)\n                cf_matrix.append(\"FP\")\n        for gt in gts:\n            score = -1\n            for pd in pds:\n                js = self.jaccard(gt, pd)\n                if js > score:\n                    score = js\n            if score == 0:\n                js_scores.append(score)\n                cf_matrix.append(\"FN\")     \n        return js_scores, \" \".join(cf_matrix)\n\n    def get_count_tp_fp_fn(self,prediction, verbose=True):\n        preds = prediction.split(\" \")\n        if verbose:\n            print(preds)\n        tpc = 0\n        fpc = 0\n        fnc = 0\n        for pred in preds:\n            if pred == \"TP\":\n                tpc = tpc + 1\n            elif pred == \"FP\":\n                fpc = fpc + 1\n            elif pred == \"FN\":\n                fnc = fnc + 1\n        return [tpc, fpc, fnc]\n\n    def make_col_tp_fp_fn(self,df, col):\n        df['TP'] = df[col].apply(lambda x : x[0])\n        df['FP'] = df[col].apply(lambda x : x[1])\n        df['FN'] = df[col].apply(lambda x : x[2])\n        return df","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:06.997418Z","iopub.execute_input":"2021-06-14T08:52:06.997816Z","iopub.status.idle":"2021-06-14T08:52:07.020918Z","shell.execute_reply.started":"2021-06-14T08:52:06.997776Z","shell.execute_reply":"2021-06-14T08:52:07.018838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMISSION:\n    print(\"no evaluation possible\")\nelse:\n    scoring_result = Score(result, \"label\", \"PredictionString\")\n    print(scoring_result.final_score)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:52:07.022602Z","iopub.execute_input":"2021-06-14T08:52:07.023010Z","iopub.status.idle":"2021-06-14T08:52:07.350167Z","shell.execute_reply.started":"2021-06-14T08:52:07.022971Z","shell.execute_reply":"2021-06-14T08:52:07.349162Z"},"trusted":true},"execution_count":null,"outputs":[]}]}