{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a simple baseline for NER detection for Coleridge competition . Here the idea is to train a simple Spacy phrase matcher to identify the labels which we encounter in the train set to test set. This can also generalize into a very strong final solution if you are looking at small subset of dataset to recognize in your data. \n\nThis notebook scores 0.48 in F1 metric in private test set. ","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport os \nimport json\nimport re\nimport spacy\nfrom spacy.matcher import PhraseMatcher\nfrom spacy.lang.en import English","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:37.648909Z","iopub.execute_input":"2021-06-03T22:41:37.649440Z","iopub.status.idle":"2021-06-03T22:41:37.654182Z","shell.execute_reply.started":"2021-06-03T22:41:37.649393Z","shell.execute_reply":"2021-06-03T22:41:37.653154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\nnlp = spacy.load(\"en_core_web_sm\")\nnlp.max_length = 150000000\n\nterms=df['dataset_label'].unique().tolist()\n\nmatcher = PhraseMatcher(nlp.vocab)\n\n# Only run nlp.make_doc to speed things up\npatterns = [nlp.make_doc(text) for text in terms]\nmatcher.add('Dataset_Label',patterns)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:37.655557Z","iopub.execute_input":"2021-06-03T22:41:37.655845Z","iopub.status.idle":"2021-06-03T22:41:39.111790Z","shell.execute_reply.started":"2021-06-03T22:41:37.655818Z","shell.execute_reply":"2021-06-03T22:41:39.110683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:39.113520Z","iopub.execute_input":"2021-06-03T22:41:39.113841Z","iopub.status.idle":"2021-06-03T22:41:39.118636Z","shell.execute_reply.started":"2021-06-03T22:41:39.113810Z","shell.execute_reply":"2021-06-03T22:41:39.117633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_root='../input/coleridgeinitiative-show-us-the-data/test'\ntest_files=os.listdir('../input/coleridgeinitiative-show-us-the-data/test')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:39.120154Z","iopub.execute_input":"2021-06-03T22:41:39.120446Z","iopub.status.idle":"2021-06-03T22:41:39.139305Z","shell.execute_reply.started":"2021-06-03T22:41:39.120402Z","shell.execute_reply":"2021-06-03T22:41:39.138213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DATA = dict()\n\nfor file in test_files:\n    with open(os.path.join(test_root,file), encoding=\"utf8\") as f:\n        TEXTS = json.loads(f.read())\n\n        \n    file_list=[]\n        \n    for i,(key,value) in enumerate(TEXTS):\n        file_list.append(TEXTS[i][value])\n    \n    TEST_DATA[file.split('.')[0]]=file_list","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:39.140785Z","iopub.execute_input":"2021-06-03T22:41:39.141207Z","iopub.status.idle":"2021-06-03T22:41:39.179830Z","shell.execute_reply.started":"2021-06-03T22:41:39.141163Z","shell.execute_reply":"2021-06-03T22:41:39.178806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels=dict()\n\nfor key,item in TEST_DATA.items():\n    labels=[]\n    for i in np.arange(len(item)):\n        \n        if (len(TEST_DATA[key][i])<100000):\n            doc=nlp(TEST_DATA[key][i],disable=[\"tagger\", \"parser\",'ner','textcat','lemmatizer'])\n            matches = matcher(doc)\n            for match_id, start, end in matches:\n                span = doc[start:end]  # The matched span\n                labels.append(clean_text(span.text))\n  \n            \n    labels=set(labels)\n    test_labels[key]=\"|\".join(str(e) for e in labels)\n        \ndf = pd.DataFrame(list(test_labels.items()),columns = ['Id','PredictionString']) \n    \ndf['PredictionString'].fillna('alzheimer s disease neuroimaging initiative',inplace=True)\n\ndf.to_csv('./submission.csv',index=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:39.181135Z","iopub.execute_input":"2021-06-03T22:41:39.181442Z","iopub.status.idle":"2021-06-03T22:41:39.592659Z","shell.execute_reply.started":"2021-06-03T22:41:39.181392Z","shell.execute_reply":"2021-06-03T22:41:39.591615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:41:39.593917Z","iopub.execute_input":"2021-06-03T22:41:39.594200Z","iopub.status.idle":"2021-06-03T22:41:39.613837Z","shell.execute_reply.started":"2021-06-03T22:41:39.594173Z","shell.execute_reply":"2021-06-03T22:41:39.612773Z"},"trusted":true},"execution_count":null,"outputs":[]}]}