{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\nBrif data exploration. Rule-based Spacy model (pattern matching) as a baseline.\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-22T11:35:09.915384Z","iopub.execute_input":"2021-08-22T11:35:09.915810Z","iopub.status.idle":"2021-08-22T11:35:10.796054Z","shell.execute_reply.started":"2021-08-22T11:35:09.915724Z","shell.execute_reply":"2021-08-22T11:35:10.795064Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport re\n\nimport json\nfrom pathlib import Path\nfrom collections import OrderedDict\n\nfrom typing import List, Dict, Set\nfrom thinc.api import Model\n\nimport spacy\nfrom spacy.tokens import Doc\nfrom spacy.matcher import Matcher\nfrom spacy import displacy\n\nfrom nltk import tokenize\nfrom tqdm import tqdm\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:42.885809Z","iopub.execute_input":"2021-08-22T14:23:42.886130Z","iopub.status.idle":"2021-08-22T14:23:42.893359Z","shell.execute_reply.started":"2021-08-22T14:23:42.886100Z","shell.execute_reply":"2021-08-22T14:23:42.891993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data\n\nPublications are provided in JSON format, broken up into sections with section titles. The goal in [this competition](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview) is not just to match known dataset strings but to generalize to datasets that have never been seen before using NLP and statistical techniques. A percentage of the public test set publications are drawn from the training set - not all datasets have been identified in train, so these unidentified datasets have been used as a portion of the public test labels. These should serve as guides for the difficult task of labeling the private test set. [Details](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/data)","metadata":{}},{"cell_type":"code","source":"TRAIN_PATH = '../input/coleridgeinitiative-show-us-the-data/train/'\nTEST_PATH = '../input/coleridgeinitiative-show-us-the-data/test/'","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:42.894803Z","iopub.execute_input":"2021-08-22T14:23:42.895069Z","iopub.status.idle":"2021-08-22T14:23:42.911112Z","shell.execute_reply.started":"2021-08-22T14:23:42.895042Z","shell.execute_reply":"2021-08-22T14:23:42.910280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:42.912740Z","iopub.execute_input":"2021-08-22T14:23:42.913000Z","iopub.status.idle":"2021-08-22T14:23:43.024357Z","shell.execute_reply.started":"2021-08-22T14:23:42.912974Z","shell.execute_reply":"2021-08-22T14:23:43.023330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:43.025957Z","iopub.execute_input":"2021-08-22T14:23:43.026284Z","iopub.status.idle":"2021-08-22T14:23:43.040583Z","shell.execute_reply.started":"2021-08-22T14:23:43.026251Z","shell.execute_reply":"2021-08-22T14:23:43.039597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.select_dtypes(include=[object]).describe().T","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:43.041821Z","iopub.execute_input":"2021-08-22T14:23:43.042391Z","iopub.status.idle":"2021-08-22T14:23:43.106271Z","shell.execute_reply.started":"2021-08-22T14:23:43.042346Z","shell.execute_reply":"2021-08-22T14:23:43.105280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Id & Unique Labels","metadata":{}},{"cell_type":"code","source":"# Get all labels for each Id \ndef get_labels(df: pd.DataFrame) -> Dict[str, Set[str]]:\n    ids = df.Id.unique().tolist()\n    output = {}\n    for i in ids:\n        labels = df.loc[df.Id == i]['cleaned_label'].unique().tolist()\n        output[i] = labels\n\n    return output","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:43.108613Z","iopub.execute_input":"2021-08-22T14:23:43.109001Z","iopub.status.idle":"2021-08-22T14:23:43.115034Z","shell.execute_reply.started":"2021-08-22T14:23:43.108958Z","shell.execute_reply":"2021-08-22T14:23:43.113819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nid_labels = get_labels(df)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:43.116856Z","iopub.execute_input":"2021-08-22T14:23:43.117334Z","iopub.status.idle":"2021-08-22T14:24:09.111066Z","shell.execute_reply.started":"2021-08-22T14:23:43.117292Z","shell.execute_reply":"2021-08-22T14:24:09.109929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(id_labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.112333Z","iopub.execute_input":"2021-08-22T14:24:09.112593Z","iopub.status.idle":"2021-08-22T14:24:09.118405Z","shell.execute_reply.started":"2021-08-22T14:24:09.112566Z","shell.execute_reply":"2021-08-22T14:24:09.117310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_labels['d0fa7568-7d8e-4db9-870f-f9c6f668c17b']","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.119848Z","iopub.execute_input":"2021-08-22T14:24:09.120141Z","iopub.status.idle":"2021-08-22T14:24:09.133153Z","shell.execute_reply.started":"2021-08-22T14:24:09.120111Z","shell.execute_reply":"2021-08-22T14:24:09.132230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Patterns","metadata":{}},{"cell_type":"markdown","source":"```python\npattern_sample = [\n    {'LOWER': 'national'},\n    {'LOWER': 'education'},\n    {'LOWER': 'longitudinal'},\n    {'LOWER': 'study'}\n]\n```","metadata":{"execution":{"iopub.status.busy":"2021-08-19T14:57:57.555425Z","iopub.execute_input":"2021-08-19T14:57:57.555771Z","iopub.status.idle":"2021-08-19T14:57:57.562098Z","shell.execute_reply.started":"2021-08-19T14:57:57.555733Z","shell.execute_reply":"2021-08-19T14:57:57.560919Z"}}},{"cell_type":"code","source":"BasePattern = List[Dict[str, str]]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.134537Z","iopub.execute_input":"2021-08-22T14:24:09.134903Z","iopub.status.idle":"2021-08-22T14:24:09.143270Z","shell.execute_reply.started":"2021-08-22T14:24:09.134873Z","shell.execute_reply":"2021-08-22T14:24:09.142175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleaned label to label abbreviation \ndef get_abbreviation(words: List[str]) -> str:\n    return ''.join([i[0].upper() for i in words])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.144519Z","iopub.execute_input":"2021-08-22T14:24:09.144883Z","iopub.status.idle":"2021-08-22T14:24:09.155946Z","shell.execute_reply.started":"2021-08-22T14:24:09.144854Z","shell.execute_reply":"2021-08-22T14:24:09.154784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_pattern(label: str) -> Dict[str, BasePattern]:\n    words = label.split()\n    pattern = []\n    for word in words:\n        lower = {}\n        lower['LOWER'] = word.lower()\n        pattern.append(lower)\n    return {label: pattern}","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.157328Z","iopub.execute_input":"2021-08-22T14:24:09.157634Z","iopub.status.idle":"2021-08-22T14:24:09.168909Z","shell.execute_reply.started":"2021-08-22T14:24:09.157606Z","shell.execute_reply":"2021-08-22T14:24:09.167797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fetch_pattern('Hello World')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.170404Z","iopub.execute_input":"2021-08-22T14:24:09.170726Z","iopub.status.idle":"2021-08-22T14:24:09.182834Z","shell.execute_reply.started":"2021-08-22T14:24:09.170697Z","shell.execute_reply":"2021-08-22T14:24:09.181900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_patterns(labels: List[str]) -> List[Dict[str, BasePattern]]:\n    patterns = []\n    for label in labels:\n        pattern = fetch_pattern(label)\n        patterns.append(pattern)        \n    return patterns","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.185887Z","iopub.execute_input":"2021-08-22T14:24:09.186244Z","iopub.status.idle":"2021-08-22T14:24:09.195465Z","shell.execute_reply.started":"2021-08-22T14:24:09.186211Z","shell.execute_reply":"2021-08-22T14:24:09.194458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = df.cleaned_label.unique().tolist()\nlen(labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.197625Z","iopub.execute_input":"2021-08-22T14:24:09.198003Z","iopub.status.idle":"2021-08-22T14:24:09.212632Z","shell.execute_reply.started":"2021-08-22T14:24:09.197961Z","shell.execute_reply":"2021-08-22T14:24:09.211650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPATTERNS = create_patterns(labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.215545Z","iopub.execute_input":"2021-08-22T14:24:09.215835Z","iopub.status.idle":"2021-08-22T14:24:09.225556Z","shell.execute_reply.started":"2021-08-22T14:24:09.215807Z","shell.execute_reply":"2021-08-22T14:24:09.224747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATTERNS[:5]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.226650Z","iopub.execute_input":"2021-08-22T14:24:09.226899Z","iopub.status.idle":"2021-08-22T14:24:09.240020Z","shell.execute_reply.started":"2021-08-22T14:24:09.226874Z","shell.execute_reply":"2021-08-22T14:24:09.239384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matcher","metadata":{}},{"cell_type":"code","source":"model_name = \"en_core_web_sm\"\nnlp = spacy.load(model_name, disable=[\"tagger\", \"parser\", \"ner\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.240894Z","iopub.execute_input":"2021-08-22T14:24:09.241247Z","iopub.status.idle":"2021-08-22T14:24:09.706635Z","shell.execute_reply.started":"2021-08-22T14:24:09.241220Z","shell.execute_reply":"2021-08-22T14:24:09.705842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add created patterns to Matcher\ndef add_patterns(model: Model) -> Matcher:\n    matcher = Matcher(model.vocab, validate=True)\n    for name_pattern in PATTERNS:\n        for name, pattern in name_pattern.items():\n            matcher.add(name, [pattern], on_match=None)\n    return matcher","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.707653Z","iopub.execute_input":"2021-08-22T14:24:09.708040Z","iopub.status.idle":"2021-08-22T14:24:09.712041Z","shell.execute_reply.started":"2021-08-22T14:24:09.708011Z","shell.execute_reply":"2021-08-22T14:24:09.711412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matcher = add_patterns(nlp)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.712950Z","iopub.execute_input":"2021-08-22T14:24:09.713316Z","iopub.status.idle":"2021-08-22T14:24:09.755747Z","shell.execute_reply.started":"2021-08-22T14:24:09.713289Z","shell.execute_reply":"2021-08-22T14:24:09.754705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric - Micro FBeta Score","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/230091\n# https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\ndef compute_fbeta(y_true: List[List[str]],\n                  y_pred: List[List[str]],\n                  beta: float = 0.5) -> float:\n    \"\"\"Compute the Jaccard-based micro FBeta score.\n    \"\"\"\n\n    def _jaccard_similarity(str1: str, str2: str) -> float:\n        a = set(str1.split()) \n        b = set(str2.split())\n        c = a.intersection(b)\n        return float(len(c)) / (len(a) + len(b) - len(c))\n\n    tp = 0  # true positive\n    fp = 0  # false positive\n    fn = 0  # false negative\n    for ground_truth_list, predicted_string_list in zip(y_true, y_pred):\n        predicted_string_list_sorted = sorted(predicted_string_list)\n        for ground_truth in sorted(ground_truth_list):            \n            if len(predicted_string_list_sorted) == 0:\n                fn += 1\n            else:\n                similarity_scores = [\n                    _jaccard_similarity(ground_truth, predicted_string)\n                    for predicted_string in predicted_string_list_sorted\n                ]\n                matched_idx = np.argmax(similarity_scores)\n                if similarity_scores[matched_idx] >= 0.5:\n                    predicted_string_list_sorted.pop(matched_idx)\n                    tp += 1\n                else:\n                    fn += 1\n        fp += len(predicted_string_list_sorted)\n\n    tp *= (1 + beta ** 2)\n    fn *= beta ** 2\n    fbeta_score = tp / (tp + fp + fn)\n    return fbeta_score","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.756900Z","iopub.execute_input":"2021-08-22T14:24:09.757159Z","iopub.status.idle":"2021-08-22T14:24:09.766105Z","shell.execute_reply.started":"2021-08-22T14:24:09.757133Z","shell.execute_reply":"2021-08-22T14:24:09.765181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check metric sanity\ncompute_fbeta(\n    [['national education longitudinal study','slosh model']],\n    [['education', 'slosh model']]\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.767162Z","iopub.execute_input":"2021-08-22T14:24:09.767627Z","iopub.status.idle":"2021-08-22T14:24:09.779455Z","shell.execute_reply.started":"2021-08-22T14:24:09.767586Z","shell.execute_reply":"2021-08-22T14:24:09.778715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"def read_json(path: str, pub_id: str) -> None:\n    path = Path(path + pub_id + '.json')\n    with open(path, 'rt') as json_file:\n        return json.load(json_file, object_hook=OrderedDict)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.780528Z","iopub.execute_input":"2021-08-22T14:24:09.780929Z","iopub.status.idle":"2021-08-22T14:24:09.795100Z","shell.execute_reply.started":"2021-08-22T14:24:09.780887Z","shell.execute_reply":"2021-08-22T14:24:09.794421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.796196Z","iopub.execute_input":"2021-08-22T14:24:09.796617Z","iopub.status.idle":"2021-08-22T14:24:09.807970Z","shell.execute_reply.started":"2021-08-22T14:24:09.796575Z","shell.execute_reply":"2021-08-22T14:24:09.807280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_patterns(docs: Doc) -> List[str]:\n    labels = set()\n    for doc in docs:\n        for match_id, start, end in matcher(doc):\n            label = doc.vocab.strings[match_id]\n            labels.add(label)\n    if not labels:\n        labels.add('')\n    return list(labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.808980Z","iopub.execute_input":"2021-08-22T14:24:09.809415Z","iopub.status.idle":"2021-08-22T14:24:09.820277Z","shell.execute_reply.started":"2021-08-22T14:24:09.809373Z","shell.execute_reply":"2021-08-22T14:24:09.819568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_preds(ids: List[str], path: str, model: Model) -> Dict[str, List[str]]:\n    \n    output = {}\n    for pub_id in tqdm(ids):\n        json_file = read_json(path, pub_id)\n        \n        # JSON data to the list of lists [sentences of each section of the publication]\n        sentences = [tokenize.sent_tokenize(passage['text']) for passage in json_file]\n        \n        # Flattening the list to create a list on all sentences for each publication Id\n        flatten = lambda lst: [clean_text(item) for sublist in lst for item in sublist]\n        sentences = flatten(sentences)\n        \n        # Get all Doc objects to operate on sentence level\n        docs = []\n        for sent in sentences:\n            doc = model(sent)\n            docs.append(doc) \n            \n        # Rule-based matching\n        output[pub_id] = find_patterns(docs)\n    gc.collect()\n        \n    return output","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.821357Z","iopub.execute_input":"2021-08-22T14:24:09.821785Z","iopub.status.idle":"2021-08-22T14:24:09.834379Z","shell.execute_reply.started":"2021-08-22T14:24:09.821743Z","shell.execute_reply":"2021-08-22T14:24:09.833129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = df.Id.unique().tolist()\nlen(ids)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.835808Z","iopub.execute_input":"2021-08-22T14:24:09.836422Z","iopub.status.idle":"2021-08-22T14:24:09.852754Z","shell.execute_reply.started":"2021-08-22T14:24:09.836374Z","shell.execute_reply":"2021-08-22T14:24:09.851738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the output format of the model\nget_preds(ids[:2], TRAIN_PATH, nlp)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:09.853831Z","iopub.execute_input":"2021-08-22T14:24:09.854140Z","iopub.status.idle":"2021-08-22T14:24:10.050228Z","shell.execute_reply.started":"2021-08-22T14:24:09.854109Z","shell.execute_reply":"2021-08-22T14:24:10.049328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predictions from pattern matching model\npreds = get_preds(ids[:1000], TRAIN_PATH, nlp)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:10.051374Z","iopub.execute_input":"2021-08-22T14:24:10.051647Z","iopub.status.idle":"2021-08-22T14:25:18.638613Z","shell.execute_reply.started":"2021-08-22T14:24:10.051619Z","shell.execute_reply":"2021-08-22T14:25:18.637685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation ","metadata":{}},{"cell_type":"code","source":"def filter_ids(id_labels: Dict[str, List[str]], ids: List[str]) -> Dict[str, List[str]]:\n    return {pub_id: id_labels[pub_id] for pub_id in ids}","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:18.639802Z","iopub.execute_input":"2021-08-22T14:25:18.640065Z","iopub.status.idle":"2021-08-22T14:25:18.644559Z","shell.execute_reply.started":"2021-08-22T14:25:18.640040Z","shell.execute_reply":"2021-08-22T14:25:18.643524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the filtering output\nfilter_ids(id_labels, ids[:2])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:18.645562Z","iopub.execute_input":"2021-08-22T14:25:18.645855Z","iopub.status.idle":"2021-08-22T14:25:18.661729Z","shell.execute_reply.started":"2021-08-22T14:25:18.645828Z","shell.execute_reply":"2021-08-22T14:25:18.660823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# References for the evaluation\nrefs = filter_ids(id_labels, ids[:1000])\nlen(preds), len(refs)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:18.662910Z","iopub.execute_input":"2021-08-22T14:25:18.663172Z","iopub.status.idle":"2021-08-22T14:25:18.676318Z","shell.execute_reply.started":"2021-08-22T14:25:18.663146Z","shell.execute_reply":"2021-08-22T14:25:18.675267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ref_list = [v for k,v in refs.items()]\npred_list = [v for k,v in preds.items()]\nfbeta = compute_fbeta(ref_list, pred_list)\nprint(f'FBeta Score for sample of {len(preds)} publications: {fbeta:.4f}')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:18.677545Z","iopub.execute_input":"2021-08-22T14:25:18.677842Z","iopub.status.idle":"2021-08-22T14:25:18.718523Z","shell.execute_reply.started":"2021-08-22T14:25:18.677813Z","shell.execute_reply":"2021-08-22T14:25:18.717664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntest_ids = test.Id.to_list()\ntest_preds = get_preds(test_ids, TEST_PATH, nlp)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:18.719586Z","iopub.execute_input":"2021-08-22T14:25:18.719849Z","iopub.status.idle":"2021-08-22T14:25:19.421761Z","shell.execute_reply.started":"2021-08-22T14:25:18.719823Z","shell.execute_reply":"2021-08-22T14:25:19.420778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {'Id': [], 'PredictionString': []}\nfor i, labels in test_preds.items():\n    data['Id'].append(i)\n    labels = '|'.join(labels)\n    data['PredictionString'].append(labels)\n    \nsub = pd.DataFrame.from_dict(data)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:19.423171Z","iopub.execute_input":"2021-08-22T14:25:19.423504Z","iopub.status.idle":"2021-08-22T14:25:19.431252Z","shell.execute_reply.started":"2021-08-22T14:25:19.423474Z","shell.execute_reply":"2021-08-22T14:25:19.430231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('./submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:19.432590Z","iopub.execute_input":"2021-08-22T14:25:19.432962Z","iopub.status.idle":"2021-08-22T14:25:19.453129Z","shell.execute_reply.started":"2021-08-22T14:25:19.432933Z","shell.execute_reply":"2021-08-22T14:25:19.452108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Notes\n\n## Research\n\n- [ ] Study the inconsistencies to expand the pattern types.\n- [ ] The publications in the training dataset are poorly labeled, the test set seems to have many more labels than there are in the train set. Find external public data to extract more labels to better generalize to datasets so improve the rule-based and statistical models.\n- [ ] Review possible ML approaches for automatic data labeling.\n\n## Code\n\n- [x] Basic Patterns\n- [x] Spacy Matcher\n- [x] Improve Metric\n- [x] Review annotations\n- [ ] Add docs","metadata":{}}]}