{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:01:33.292271Z","iopub.execute_input":"2021-06-05T15:01:33.292701Z","iopub.status.idle":"2021-06-05T15:01:33.29748Z","shell.execute_reply.started":"2021-06-05T15:01:33.292613Z","shell.execute_reply":"2021-06-05T15:01:33.29657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import dump, load\n\npos_vectorizer = load(\"../input/preselection-svc/pos_vectorizer.joblib\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:01:33.307705Z","iopub.execute_input":"2021-06-05T15:01:33.308156Z","iopub.status.idle":"2021-06-05T15:01:34.376584Z","shell.execute_reply.started":"2021-06-05T15:01:33.308114Z","shell.execute_reply":"2021-06-05T15:01:34.375658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:01:34.379345Z","iopub.execute_input":"2021-06-05T15:01:34.379664Z","iopub.status.idle":"2021-06-05T15:03:07.768889Z","shell.execute_reply.started":"2021-06-05T15:01:34.379634Z","shell.execute_reply":"2021-06-05T15:03:07.767692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_colwidth', None)\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-05T15:03:07.770684Z","iopub.execute_input":"2021-06-05T15:03:07.771001Z","iopub.status.idle":"2021-06-05T15:03:07.893297Z","shell.execute_reply.started":"2021-06-05T15:03:07.770971Z","shell.execute_reply":"2021-06-05T15:03:07.892151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\ntest_sample = train.sample(100)\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:03:07.894516Z","iopub.execute_input":"2021-06-05T15:03:07.894798Z","iopub.status.idle":"2021-06-05T15:04:10.953258Z","shell.execute_reply.started":"2021-06-05T15:03:07.894772Z","shell.execute_reply":"2021-06-05T15:04:10.952284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:04:10.957017Z","iopub.execute_input":"2021-06-05T15:04:10.9574Z","iopub.status.idle":"2021-06-05T15:04:10.998456Z","shell.execute_reply.started":"2021-06-05T15:04:10.957365Z","shell.execute_reply":"2021-06-05T15:04:10.99755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preselection model","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:04:11.000272Z","iopub.execute_input":"2021-06-05T15:04:11.000741Z","iopub.status.idle":"2021-06-05T15:04:11.006978Z","shell.execute_reply.started":"2021-06-05T15:04:11.000709Z","shell.execute_reply":"2021-06-05T15:04:11.00545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport nltk\nids = train['Id'].values\ncontents = []\nfor idx, row in enumerate(train.values):\n    print(idx, end='\\r')\n    paper_id = row[0]\n    paper_cont = []\n    for content_elem in papers[paper_id]:\n        paper_cont.append(content_elem['text'])\n\n    contents.append(\" \".join(paper_cont))        \ncombined_train = train.copy()\ncombined_train['contents'] = contents\ncombined_train","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:04:11.0096Z","iopub.execute_input":"2021-06-05T15:04:11.010268Z","iopub.status.idle":"2021-06-05T15:04:15.543666Z","shell.execute_reply.started":"2021-06-05T15:04:11.010217Z","shell.execute_reply":"2021-06-05T15:04:15.542196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_contents = []\nlabels = []\nfor idx, (label, contents) in enumerate(combined_train[['cleaned_label', 'contents']].values):\n    print(idx, end='\\r')\n    sentences = list(nltk.sent_tokenize(contents))\n    sentence_labels = []\n    for sentence in sentences:\n        if all(clean_text(word) in clean_text(sentence) for word in label.split()):\n            sentence_labels.append(1.0)\n        else:\n            sentence_labels.append(0.0)\n    new_contents.extend(sentences)\n    labels.extend(sentence_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:04:15.545004Z","iopub.execute_input":"2021-06-05T15:04:15.545355Z","iopub.status.idle":"2021-06-05T15:16:18.207541Z","shell.execute_reply.started":"2021-06-05T15:04:15.545324Z","shell.execute_reply":"2021-06-05T15:16:18.205787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preselection_df = pd.DataFrame({'sentence': new_contents, 'label': labels})","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:18.209678Z","iopub.execute_input":"2021-06-05T15:16:18.210027Z","iopub.status.idle":"2021-06-05T15:16:21.235821Z","shell.execute_reply.started":"2021-06-05T15:16:18.209994Z","shell.execute_reply":"2021-06-05T15:16:21.23451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preselection_short = preselection_df.sample(frac=0.05, weights = 1./preselection_df.groupby('label')['label'].transform('count'))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:21.237492Z","iopub.execute_input":"2021-06-05T15:16:21.237815Z","iopub.status.idle":"2021-06-05T15:16:22.603903Z","shell.execute_reply.started":"2021-06-05T15:16:21.237786Z","shell.execute_reply":"2021-06-05T15:16:22.602746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preselection_short.groupby('label').count()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:22.605414Z","iopub.execute_input":"2021-06-05T15:16:22.605727Z","iopub.status.idle":"2021-06-05T15:16:22.775442Z","shell.execute_reply.started":"2021-06-05T15:16:22.605698Z","shell.execute_reply":"2021-06-05T15:16:22.774322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(\n    preselection_short['sentence'].values, preselection_short['label'].values, test_size=0.25, random_state=99)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:22.776911Z","iopub.execute_input":"2021-06-05T15:16:22.777232Z","iopub.status.idle":"2021-06-05T15:16:22.851932Z","shell.execute_reply.started":"2021-06-05T15:16:22.777203Z","shell.execute_reply":"2021-06-05T15:16:22.850849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef pos_convert_text(sentence):\n    \"\"\"Helper code to compute average word length of a name\"\"\"\n    converted_data = []\n    sentence_tags = []\n    tags = nltk.pos_tag(sentence.split())\n    for tag in tags:\n        sentence_tags.append(tag[1])\n    converted_data.append(\" \".join(sentence_tags).replace(\".\",\"\"))\n    return converted_data\n","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:22.900986Z","iopub.execute_input":"2021-06-05T15:16:22.901459Z","iopub.status.idle":"2021-06-05T15:16:22.920964Z","shell.execute_reply.started":"2021-06-05T15:16:22.901408Z","shell.execute_reply":"2021-06-05T15:16:22.91963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:22.970177Z","iopub.execute_input":"2021-06-05T15:16:22.970655Z","iopub.status.idle":"2021-06-05T15:16:23.450426Z","shell.execute_reply.started":"2021-06-05T15:16:22.970598Z","shell.execute_reply":"2021-06-05T15:16:23.449121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(sentence):\n    pos_sentence = pos_convert_text(row)\n    features = pos_vectorizer.transform(pos_sentence).toarray().reshape(-1,14)\n    capitals_frac = sum([1 for c in row if c.isupper()])/len(row)\n    features = np.append(features, capitals_frac)\n    return features","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:23.451872Z","iopub.execute_input":"2021-06-05T15:16:23.452251Z","iopub.status.idle":"2021-06-05T15:16:23.463778Z","shell.execute_reply.started":"2021-06-05T15:16:23.452167Z","shell.execute_reply":"2021-06-05T15:16:23.462587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nfrom sklearn.svm import SVC\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\n\nstop_words = set(stopwords.words('english'))\ntrain_features = []\ntrain_labels = []\nclf = make_pipeline(StandardScaler(), SVC(gamma='auto', class_weight = 'balanced', verbose=True), verbose=True)\nfor idx, (row, label) in enumerate(zip(X_train, y_train)):\n    print(f\"{idx} / {X_train.shape[0]}\", end='\\r')\n    features = get_features(row)\n    if(features[-1] < 0.25):\n        train_features.append(features)\n        train_labels.append(label)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:16:23.465568Z","iopub.execute_input":"2021-06-05T15:16:23.465931Z","iopub.status.idle":"2021-06-05T15:30:33.447404Z","shell.execute_reply.started":"2021-06-05T15:16:23.465896Z","shell.execute_reply":"2021-06-05T15:30:33.446254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\ntrain_features = np.array(train_features)\n\nclf = make_pipeline(StandardScaler(), RandomForestClassifier(), verbose=True)\nclf.fit(train_features, train_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T13:17:53.830984Z","iopub.execute_input":"2021-06-03T13:17:53.831344Z","iopub.status.idle":"2021-06-03T14:06:23.162246Z","shell.execute_reply.started":"2021-06-03T13:17:53.831302Z","shell.execute_reply":"2021-06-03T14:06:23.160848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import dump, load\nimport os\ndump(clf, 'presel_svc.joblib') ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:06:23.164699Z","iopub.execute_input":"2021-06-03T14:06:23.165198Z","iopub.status.idle":"2021-06-03T14:06:23.214732Z","shell.execute_reply.started":"2021-06-03T14:06:23.165159Z","shell.execute_reply":"2021-06-03T14:06:23.213257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\npredictions = []\nsamples = random.sample(range(len(X_test)), 10000)\nfor idx, row in enumerate(X_test):\n    print(f\"{idx} / {X_test.shape[0]}\", end='\\r')\n    features = get_features(row)\n    predictions.append(clf.predict([features]))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:06:23.217428Z","iopub.execute_input":"2021-06-03T14:06:23.218091Z","iopub.status.idle":"2021-06-03T14:21:31.888276Z","shell.execute_reply.started":"2021-06-03T14:06:23.218017Z","shell.execute_reply":"2021-06-03T14:21:31.887158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import fbeta_score\nfrom sklearn.metrics import confusion_matrix\nprint(fbeta_score(y_test, predictions, beta=0.5, average=None))\nprint(confusion_matrix(y_test, predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T14:21:31.891608Z","iopub.execute_input":"2021-06-03T14:21:31.892081Z","iopub.status.idle":"2021-06-03T14:21:32.691637Z","shell.execute_reply.started":"2021-06-03T14:21:31.892017Z","shell.execute_reply":"2021-06-03T14:21:32.690422Z"},"trusted":true},"execution_count":null,"outputs":[]}]}