{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n0. <a href='#imports'>Imports</a>\n1. <a href='#data'>Data loading and processing</a>\n2. <a href='#functions'>Relevant functions</a>\n3. <a href='#sentence'>Sentence level</a>\n    2. <a href='#flair'>Flair and TARS</a>\n    3. <a href='#lstm'>LSTM model</a>\n        1. <a href='#lstm_prediction'>Predict relevant sentences</a>\n        2. <a href='#lstm_model'>Save Model</a>\n","metadata":{}},{"cell_type":"markdown","source":"<a id='imports'></a>\n# Imports","metadata":{}},{"cell_type":"code","source":"%%capture\nimport numpy as np\nimport pandas as pd\nimport json\nimport os\nimport re\nimport pprint\nfrom tqdm import tqdm\nimport spacy\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping\nfrom keras.models import Model\nfrom keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Input, GlobalAveragePooling1D\n\nfrom collections import Counter\n\nimport nltk\nimport tensorflow as tf\nfrom fuzzywuzzy import fuzz\n\n!pip install --upgrade git+https://github.com/zalandoresearch/flair.git\n    \nfrom flair.data import Corpus\nfrom flair.datasets import SentenceDataset\nfrom flair.trainers import ModelTrainer\nfrom flair.models.text_classification_model import TARSClassifier\nfrom flair.data import Sentence\n\nfrom sklearn.datasets import make_classification\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_recall_curve\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import auc, roc_curve\nfrom numpy import argmax\nimport scipy.stats","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-21T11:58:28.612877Z","iopub.execute_input":"2021-05-21T11:58:28.613268Z","iopub.status.idle":"2021-05-21T11:59:32.031427Z","shell.execute_reply.started":"2021-05-21T11:58:28.613237Z","shell.execute_reply":"2021-05-21T11:59:32.029989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='data'></a>\n# Data loading and processing","metadata":{}},{"cell_type":"code","source":"train_df_annotated = pd.read_pickle(\"../input/give-us-the-data-in-sentences/train_df_annotated.pkl\")\ntrain_df_annotated.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:59:32.034458Z","iopub.execute_input":"2021-05-21T11:59:32.034932Z","iopub.status.idle":"2021-05-21T11:59:37.181107Z","shell.execute_reply.started":"2021-05-21T11:59:32.034884Z","shell.execute_reply":"2021-05-21T11:59:37.179792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_titles = set([x.lower() for x in CI_train_df['dataset_label'].unique()] + \n                [x.lower() for x in CI_train_df['dataset_title'].unique()] + \n                [x.lower() for x in CI_train_df['cleaned_label'].unique()])\nds_titles = np.array(list(ds_titles))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:59:37.330635Z","iopub.execute_input":"2021-05-21T11:59:37.331103Z","iopub.status.idle":"2021-05-21T11:59:37.349304Z","shell.execute_reply.started":"2021-05-21T11:59:37.331053Z","shell.execute_reply":"2021-05-21T11:59:37.348082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_pickle(\"../input/give-us-the-data-in-sentences/train_df.pkl\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:59:37.412352Z","iopub.execute_input":"2021-05-21T11:59:37.412643Z","iopub.status.idle":"2021-05-21T12:00:01.115331Z","shell.execute_reply.started":"2021-05-21T11:59:37.412617Z","shell.execute_reply":"2021-05-21T12:00:01.113846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = []\nfor row in tqdm(train_df.itertuples(), total = train_df.shape[0]):\n    clean_text = row.clean_text\n    titles_list = [t for t in ds_titles if t in clean_text] \n    \n    found_title = False\n    sentence = row.sentence_text\n    for title in titles_list:\n        if title in sentence:\n            found_title = True\n            group = 'TP'\n            sentences.append({'Id':row.Index,'sentence':sentence,'match':title,'group':group})\n    if not found_title:\n        group = 'N' if row.section_match == False else 'UNK'  \n        sentences.append({'Id':row.Index,'sentence':sentence,'match':None,'group':group})\n\nsentence_df = pd.DataFrame(sentences)\nsentence_df['n_chars'] = sentence_df.sentence.str.len()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.171937Z","iopub.status.idle":"2021-05-21T11:58:25.172374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df.group.unique()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:06.739968Z","iopub.execute_input":"2021-05-21T12:01:06.740362Z","iopub.status.idle":"2021-05-21T12:01:06.96704Z","shell.execute_reply.started":"2021-05-21T12:01:06.740329Z","shell.execute_reply":"2021-05-21T12:01:06.965799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:00:01.562127Z","iopub.status.idle":"2021-05-21T12:00:01.56294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='functions'></a>\n# Relevant functions","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:22.245685Z","iopub.execute_input":"2021-05-21T12:01:22.246112Z","iopub.status.idle":"2021-05-21T12:01:22.250869Z","shell.execute_reply.started":"2021-05-21T12:01:22.246075Z","shell.execute_reply":"2021-05-21T12:01:22.249793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_text(sample_id, location='/kaggle/input/coleridgeinitiative-show-us-the-data/train/'):\n    with open(location + sample_id + \".json\", \"r\") as file:\n        sample = json.loads(file.read())\n    return \" \".join([s['section_title'] + \" \"+ s['text'] for s in sample])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:22.444709Z","iopub.execute_input":"2021-05-21T12:01:22.445095Z","iopub.status.idle":"2021-05-21T12:01:22.451887Z","shell.execute_reply.started":"2021-05-21T12:01:22.445063Z","shell.execute_reply":"2021-05-21T12:01:22.450673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text, n_words=25000, sequence_length=25, tokenizer=None): \n    tokenizer = Tokenizer(num_words=n_words, filters=r'!\"#$%&()*+,-.:;<=>?@[\\]^_`{|}~', lower=True)\n    tokenizer.fit_on_texts(text) #.values\n    X = tokenizer.texts_to_sequences(text) #.values\n    X = pad_sequences(X, maxlen=sequence_length, padding='pre')\n    return X, tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:20:20.784354Z","iopub.execute_input":"2021-05-21T12:20:20.784891Z","iopub.status.idle":"2021-05-21T12:20:20.792436Z","shell.execute_reply.started":"2021-05-21T12:20:20.784858Z","shell.execute_reply":"2021-05-21T12:20:20.790982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='sentence'></a>\n# Sentences","metadata":{}},{"cell_type":"code","source":"sentence_df.to_pickle(\"./sentence_df.pkl\")\nsentence_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:23.245834Z","iopub.execute_input":"2021-05-21T12:01:23.246209Z","iopub.status.idle":"2021-05-21T12:01:27.495157Z","shell.execute_reply.started":"2021-05-21T12:01:23.246178Z","shell.execute_reply":"2021-05-21T12:01:27.493898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Analyze sentence data","metadata":{}},{"cell_type":"code","source":"sentence_df.sample(5).append(sentence_df[sentence_df.group == 'TP'].sample(5))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:27.497351Z","iopub.execute_input":"2021-05-21T12:01:27.497754Z","iopub.status.idle":"2021-05-21T12:01:28.149959Z","shell.execute_reply.started":"2021-05-21T12:01:27.497722Z","shell.execute_reply":"2021-05-21T12:01:28.148749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mulitply_negative = 10\nTP_df = sentence_df[sentence_df.group == 'TP']\nTP_df = TP_df.append(sentence_df[sentence_df.group == 'N'].sample(TP_df.shape[0] * mulitply_negative))\nTP_df['clean'] = TP_df.sentence.str.lower().replace(r\"[^a-z ]+\",\"\", regex=True)\nTP_df['n_words'] = TP_df.clean.apply(lambda x: len(str(x).split()))\nTP_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:28.151508Z","iopub.execute_input":"2021-05-21T12:01:28.151846Z","iopub.status.idle":"2021-05-21T12:01:30.698324Z","shell.execute_reply.started":"2021-05-21T12:01:28.151815Z","shell.execute_reply":"2021-05-21T12:01:30.697222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TP_df = TP_df.drop(TP_df[TP_df.n_words < 5].n_words.index, axis=0)\nTP_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:30.699945Z","iopub.execute_input":"2021-05-21T12:01:30.700416Z","iopub.status.idle":"2021-05-21T12:01:30.78358Z","shell.execute_reply.started":"2021-05-21T12:01:30.700347Z","shell.execute_reply":"2021-05-21T12:01:30.782215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(TP_df['group'].unique())\nprint(len(TP_df[TP_df.group == 'N']))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:30.787062Z","iopub.execute_input":"2021-05-21T12:01:30.787535Z","iopub.status.idle":"2021-05-21T12:01:30.824617Z","shell.execute_reply.started":"2021-05-21T12:01:30.78748Z","shell.execute_reply":"2021-05-21T12:01:30.823061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"Maximum number of characters in sentence:\", TP_df.n_chars.max())\nprint (\"Average number of characters in sentence:\", int(TP_df.n_chars.mean()))\nprint (\"Total number of characters in sentence:\", int(TP_df.n_chars.sum()))\nTP_df[TP_df.n_chars < 1000].n_chars.hist(bins=50);","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:30.826743Z","iopub.execute_input":"2021-05-21T12:01:30.827165Z","iopub.status.idle":"2021-05-21T12:01:31.13477Z","shell.execute_reply.started":"2021-05-21T12:01:30.827122Z","shell.execute_reply":"2021-05-21T12:01:31.133836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"Maximum number of words in sentence:\", TP_df.n_words.max())\nprint (\"Average number of words in sentence:\", int(TP_df.n_words.mean()))\nprint (\"Total number of words in sentence:\", int(TP_df.n_words.sum()))\nTP_df[TP_df.n_words < 500].n_words.hist(bins=10);","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:31.136041Z","iopub.execute_input":"2021-05-21T12:01:31.13655Z","iopub.status.idle":"2021-05-21T12:01:31.323257Z","shell.execute_reply.started":"2021-05-21T12:01:31.136501Z","shell.execute_reply":"2021-05-21T12:01:31.322427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Tokenize ","metadata":{}},{"cell_type":"code","source":"%time \ntqdm.pandas()\nTP_df['tokenized'] = TP_df.clean.progress_apply(lambda x: [ \\\n    w for w in nltk.word_tokenize(x[:500])])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:31.324968Z","iopub.execute_input":"2021-05-21T12:01:31.325282Z","iopub.status.idle":"2021-05-21T12:01:55.299709Z","shell.execute_reply.started":"2021-05-21T12:01:31.325251Z","shell.execute_reply":"2021-05-21T12:01:55.298668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_words = Counter()\nfor words in tqdm(TP_df.tokenized.values):\n    unique_words.update(words)\nprint (f\"Unique words: {len(unique_words)}\")    ","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:55.30098Z","iopub.execute_input":"2021-05-21T12:01:55.30127Z","iopub.status.idle":"2021-05-21T12:01:55.847763Z","shell.execute_reply.started":"2021-05-21T12:01:55.301234Z","shell.execute_reply":"2021-05-21T12:01:55.846726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_occurencies = 10\nmin_word_len = 3\nmy_vocab = {k:v for k, v in unique_words.items() if v>=min_occurencies and len(k)>= min_word_len}\nmy_vocab = {k: v for k, v in sorted(my_vocab.items(), key=lambda item: item[1], reverse=True)}\nvocab_size = len(my_vocab)\nprint(vocab_size)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:01:55.849041Z","iopub.execute_input":"2021-05-21T12:01:55.849328Z","iopub.status.idle":"2021-05-21T12:01:55.877202Z","shell.execute_reply.started":"2021-05-21T12:01:55.849298Z","shell.execute_reply":"2021-05-21T12:01:55.876162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_words = vocab_size \nsequence_length = int(TP_df.n_chars.mean())","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:23:42.284436Z","iopub.execute_input":"2021-05-21T12:23:42.284865Z","iopub.status.idle":"2021-05-21T12:23:42.293276Z","shell.execute_reply.started":"2021-05-21T12:23:42.284823Z","shell.execute_reply":"2021-05-21T12:23:42.292341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TP_df.clean","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:25:52.497649Z","iopub.execute_input":"2021-05-21T12:25:52.498066Z","iopub.status.idle":"2021-05-21T12:25:52.508159Z","shell.execute_reply.started":"2021-05-21T12:25:52.498033Z","shell.execute_reply":"2021-05-21T12:25:52.506626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"padded_df, tokenizer_ = tokenize(TP_df.clean, sequence_length=sequence_length, n_words=vocab_size)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:23:50.832197Z","iopub.execute_input":"2021-05-21T12:23:50.833138Z","iopub.status.idle":"2021-05-21T12:23:55.870639Z","shell.execute_reply.started":"2021-05-21T12:23:50.833097Z","shell.execute_reply":"2021-05-21T12:23:55.86974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = padded_df\ny_train = (TP_df.group == 'TP').astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:02:00.89435Z","iopub.execute_input":"2021-05-21T12:02:00.895103Z","iopub.status.idle":"2021-05-21T12:02:00.912355Z","shell.execute_reply.started":"2021-05-21T12:02:00.895058Z","shell.execute_reply":"2021-05-21T12:02:00.911442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df['clean'] = sentence_df.sentence.str.lower().replace(r\"[^a-z ]+\",\"\", regex=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:02:00.913591Z","iopub.execute_input":"2021-05-21T12:02:00.913882Z","iopub.status.idle":"2021-05-21T12:02:35.651458Z","shell.execute_reply.started":"2021-05-21T12:02:00.913853Z","shell.execute_reply":"2021-05-21T12:02:35.65043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='flair'></a>\n## Flair and TARS","metadata":{}},{"cell_type":"markdown","source":"References:\n* [https://kishaloyhalder.github.io/pdfs/tars_coling2020.pdf](https://kishaloyhalder.github.io/pdfs/tars_coling2020.pdf)\n* [https://github.com/flairNLP/flair](https://github.com/flairNLP/flair)","metadata":{}},{"cell_type":"code","source":"chosen_idx_TP_0 = np.random.choice(len(TP_df[TP_df.group == 'TP']))\nprint(TP_df.clean.iloc[chosen_idx_TP_0])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.201022Z","iopub.status.idle":"2021-05-21T11:58:25.201495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_labels = TP_df.match.unique()\nprint(Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[0]].index)]))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.202361Z","iopub.status.idle":"2021-05-21T11:58:25.202839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Corpus\ntrain_corpus = SentenceDataset([\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[0]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[1]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[2]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[3]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[4]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[5]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[6]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[7]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[8]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[9]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[10]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[11]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[12]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[13]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[14]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[15]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[16]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[17]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[18]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[19]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[20]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[21]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[22]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[23]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[24]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[25]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[26]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[27]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[28]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[29]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[30]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[31]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[32]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[33]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[34]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[35]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[36]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[37]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[38]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[39]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[40]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean[np.random.choice(TP_df[TP_df.match == unique_labels[41]].index)]).add_label('TP_N', 'TP'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n    Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N')])\n\ntest_corpus = SentenceDataset([\n        Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'TP']))]).add_label('TP_N', 'TP'),\n        Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'TP']))]).add_label('TP_N', 'TP'),\n        Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N'),\n        Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))]).add_label('TP_N', 'N')])\n\ncorpus = Corpus(train=train_corpus, test=test_corpus)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.204005Z","iopub.status.idle":"2021-05-21T11:58:25.20447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\n# TARS + FLAIR\ntars = TARSClassifier.load('tars-base')\ntars.add_and_switch_to_new_task(\"TP_or_N\", label_dictionary=corpus.make_label_dictionary())","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.205523Z","iopub.status.idle":"2021-05-21T11:58:25.20597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\ntrainer = ModelTrainer(tars, corpus)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.206927Z","iopub.status.idle":"2021-05-21T11:58:25.207373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\ntrainer.train(base_path='resources/taggers/tp_or_n',\n              learning_rate=0.02,\n              mini_batch_size=1,\n              max_epochs=10,\n              train_with_dev=True,\n              )","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.208269Z","iopub.status.idle":"2021-05-21T11:58:25.208751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\n# Train model\ntars = TARSClassifier.load('resources/taggers/tp_or_n/final-model.pt')\n\n# Prepare a test sentence that includes dataset name\nsentence = Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'TP']))])\ntars.predict(sentence)\nprint(sentence)\nprint(sentence.labels[0])\n\nsentence = Sentence(TP_df.clean.iloc[np.random.choice(len(TP_df[TP_df.group == 'N']))])\ntars.predict(sentence)\nprint(sentence)\nprint(sentence.labels[0])","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.209645Z","iopub.status.idle":"2021-05-21T11:58:25.210096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\nTP_df_part = TP_df.sample(50).append(TP_df[TP_df.group == 'TP'].sample(50))#TP_df[:100]\nTP_df_part.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.211008Z","iopub.status.idle":"2021-05-21T11:58:25.211479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\nTP_df_part['prediction'] = 0\nTP_df_part.head()\n\nfor row in tqdm(TP_df_part.itertuples()):\n    idx = row.Index\n    sentence = Sentence(TP_df_part.clean[idx])\n    tars.predict(sentence)\n    prediction_i = sentence.labels[0]\n    label_i, pred_i = str(prediction_i).rstrip().split('(')\n    pred_i, _ = str(pred_i).rstrip().split(')')\n    if (TP_df_part.group[idx]=='N') & (label_i=='N'):\n        pred_i = 1-pred_i\n    elif (TP_df_part.group[idx]=='N') & (label_i=='TP'):\n        pred_i = pred_i\n    elif (TP_df_part.group[idx]=='TP') & (label_i=='TP'):\n        pred_i = pred_i\n    elif (TP_df_part.group[idx]=='TP') & (label_i=='N'):\n        pred_i = 1-pred_i\n    TP_df_part.loc[idx,'prediction'] = float(pred_i)#pred_i","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.212568Z","iopub.status.idle":"2021-05-21T11:58:25.21301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\nTP_df_part.sample(5).append(TP_df_part[TP_df_part.group == 'TP'].sample(5))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.214033Z","iopub.status.idle":"2021-05-21T11:58:25.214508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\nprint (\"Maximum prediction of sentence:\", TP_df_part.prediction.max())\nprint (\"Minimum prediction of sentence:\", TP_df_part.prediction.min())\nprint (\"Average prediction of sentence:\", TP_df_part.prediction.mean())\nTP_df_part.prediction.hist(bins=10);","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.215685Z","iopub.status.idle":"2021-05-21T11:58:25.21613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script false\nTP_df_part.to_pickle(\"./part_flair_pred.pkl\")","metadata":{"execution":{"iopub.status.busy":"2021-05-21T11:58:25.217068Z","iopub.status.idle":"2021-05-21T11:58:25.217526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='lstm'></a>\n## LSTM model","metadata":{}},{"cell_type":"code","source":"class Transformer(layers.Layer):\n    def __init__(self, embed_dim, num_heads, rate=0.1):\n        super(Transformer, self).__init__()\n        self.multihead = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.sequential = keras.Sequential([layers.Dense(32, activation=\"relu\"), layers.Dense(embed_dim),])\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-5)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-5)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        multihead_output = self.multihead(inputs, inputs)\n        multihead_output = self.dropout1(multihead_output, training=training)\n        norm_output = self.layernorm1(inputs + multihead_output)\n        sequential_output = self.sequential(norm_output)\n        drop_output = self.dropout2(sequential_output, training=training)\n        norm_output = self.layernorm2(norm_output + drop_output)\n        return norm_output\n\nclass PositionTokenEmbedding(layers.Layer):\n    def __init__(self, maxlen, vocab_size, embed_dim):\n        super(PositionTokenEmbedding, self).__init__()\n        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        x = self.token_emb(x)\n        return x + positions","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:02:35.652881Z","iopub.execute_input":"2021-05-21T12:02:35.653388Z","iopub.status.idle":"2021-05-21T12:02:35.664958Z","shell.execute_reply.started":"2021-05-21T12:02:35.653331Z","shell.execute_reply":"2021-05-21T12:02:35.664043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_dim = 32 \nnum_heads = 2 \n\nembedding_layers = PositionTokenEmbedding(sequence_length, vocab_size, embed_dim)\ntransformer_layers = Transformer(embed_dim, num_heads)\n    \ninputs = layers.Input(shape=(sequence_length,))\nx = embedding_layers(inputs)\nx = transformer_layers(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.15)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.15)(x)\noutputs = layers.Dense(2, activation=\"softmax\")(x)\n\nmodel_LSTM = keras.Model(inputs=inputs, outputs=outputs)\nmodel_LSTM.summary()\n\nmodel_LSTM.compile(loss='sparse_categorical_crossentropy',\n              optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy'])\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0,\n                patience=0, verbose=1, mode='auto', baseline=None, restore_best_weights=True)\n\nhistory_LSTM = model_LSTM.fit(x_train, y_train, epochs=1, batch_size=32, verbose=1, callbacks=callback)\nhistory_LSTM.history.values()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:02:35.666181Z","iopub.execute_input":"2021-05-21T12:02:35.666752Z","iopub.status.idle":"2021-05-21T12:05:26.670023Z","shell.execute_reply.started":"2021-05-21T12:02:35.66672Z","shell.execute_reply":"2021-05-21T12:05:26.6692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='lstm_prediction'></a>\n### Predict relevant sentences ","metadata":{}},{"cell_type":"code","source":"set_threshold = 0.75","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:05:26.671505Z","iopub.execute_input":"2021-05-21T12:05:26.671966Z","iopub.status.idle":"2021-05-21T12:05:26.675585Z","shell.execute_reply.started":"2021-05-21T12:05:26.671922Z","shell.execute_reply":"2021-05-21T12:05:26.674753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:29:21.18201Z","iopub.execute_input":"2021-05-21T12:29:21.182395Z","iopub.status.idle":"2021-05-21T12:29:21.191869Z","shell.execute_reply.started":"2021-05-21T12:29:21.182348Z","shell.execute_reply":"2021-05-21T12:29:21.190898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df['clean_str'] = sentence_df.clean.apply(str)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:33:33.987585Z","iopub.execute_input":"2021-05-21T12:33:33.988076Z","iopub.status.idle":"2021-05-21T12:33:35.112603Z","shell.execute_reply.started":"2021-05-21T12:33:33.988044Z","shell.execute_reply":"2021-05-21T12:33:35.111731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time \ntqdm.pandas()\n\ntext = list(sentence_df.clean_str)\n\npadded_LSTM, tokenizer_LSTM = tokenize(text, sequence_length=sequence_length, n_words=vocab_size) \n\nsentence_df['tokens_tf'] = tokenizer_LSTM.texts_to_sequences(text)\n\ny_pred_logits = model_LSTM.predict(padded_LSTM, batch_size=32)\n\ny_pred_idx = y_pred_logits.argmax(axis=1)\ny_pred = y_pred_logits[:,1] #second column is for dataset mention\n\nsentence_df['pred_tf'] = y_pred \n\nthreshold_pred = [1 for p in y_pred if p > set_threshold]\n\nthreshold_df = sentence_df[sentence_df['pred_tf'] > set_threshold]\n        \nprint (f\"identified {len(threshold_pred)} candidate sentences with possible known dataset titles\") #y_pred_idx.sum()\n\nthreshold_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:35:24.98452Z","iopub.execute_input":"2021-05-21T12:35:24.984911Z","iopub.status.idle":"2021-05-21T13:29:11.636161Z","shell.execute_reply.started":"2021-05-21T12:35:24.984878Z","shell.execute_reply":"2021-05-21T13:29:11.635015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold_df.to_pickle(\"./lstm_sentence_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2021-05-21T13:29:18.213669Z","iopub.execute_input":"2021-05-21T13:29:18.214015Z","iopub.status.idle":"2021-05-21T13:29:18.700659Z","shell.execute_reply.started":"2021-05-21T13:29:18.213986Z","shell.execute_reply":"2021-05-21T13:29:18.699895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df.to_pickle(\"./sentence_df_pred.pkl\")","metadata":{"execution":{"iopub.status.busy":"2021-05-21T13:29:20.936855Z","iopub.execute_input":"2021-05-21T13:29:20.937328Z","iopub.status.idle":"2021-05-21T13:29:35.883762Z","shell.execute_reply.started":"2021-05-21T13:29:20.937298Z","shell.execute_reply":"2021-05-21T13:29:35.882731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Evaluate using different thresholds","metadata":{}},{"cell_type":"code","source":"fn_diff = 0\nthreshold_i = [0.0, 0.25, 0.5, 0.75, 0.8. 0.9]\nfor i in threshold_i:\n    prediction = lstm_eval.lstm_prediction.apply(lambda x: x > i)\n    lstm_eval_thres = lstm_eval.assign(class_pred = prediction)\n    \n    y_true = lstm_eval_thres.sentence_match.to_numpy()\n    y_pred = lstm_eval_thres.lstm_prediction.to_numpy()\n\n    tp = len(lstm_eval_thres.loc[(lstm_eval_thres.sentence_match) & (lstm_eval_thres.class_pred)])\n    tn = len(lstm_eval_thres.loc[~(lstm_eval_thres.sentence_match) & ~(lstm_eval_thres.class_pred)])\n    fp = len(lstm_eval_thres.loc[~(lstm_eval_thres.sentence_match) & (lstm_eval_thres.class_pred)])\n    fn = len(lstm_eval_thres.loc[(lstm_eval_thres.sentence_match) & ~(lstm_eval_thres.class_pred)])\n    \n    print(\"threshold: \", i)\n    \n    if i == 0.:\n        fn_diff = 0\n        print(\"number of fn: \", fn, \"/ fn percentage of whole data \", fn/len(y_true))\n        print(\"fn_diff: \", fn_diff)\n    else:\n        print(\"number of fn: \", fn, \"/ fn percentage of whole data \", fn/len(y_true))\n        print(\"fn_diff: \", fn-fn_diff)\n    fn_diff = fn\n    \n    accuracy = (tp + tn) / (tp + tn + fp + fn)\n    print(\"accuracy: \", accuracy)\n    \n    print(\"--------------\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='lstm_prediction'></a>\n### Save model","metadata":{}},{"cell_type":"code","source":"model_LSTM.save(\"./\")","metadata":{"execution":{"iopub.status.busy":"2021-05-21T13:29:35.885729Z","iopub.execute_input":"2021-05-21T13:29:35.886121Z","iopub.status.idle":"2021-05-21T13:29:40.192443Z","shell.execute_reply.started":"2021-05-21T13:29:35.88608Z","shell.execute_reply":"2021-05-21T13:29:40.191504Z"},"trusted":true},"execution_count":null,"outputs":[]}]}