{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport json\nimport re\nfrom functools import partial\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Training notebook here: https://www.kaggle.com/shivammiglani/spacy-3-0-transformer-custom-ner\n- Does not run within the time limit with all sentences.\n- Limited the sentences to be processed by spacy using `keywords`","metadata":{}},{"cell_type":"code","source":"!pip uninstall spacy -y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U spacy[transformers] --no-index --find-links=file:///kaggle/input/spacytransformers306/spacy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predefined function for preprocessing. \n\nLowercasing the dataset will make it harder to identify Entities, therefore, keeping case via clean_text2\n\nAlso not changing the index (position) of words in a sentence by other preprocessing to search for exact labels","metadata":{}},{"cell_type":"code","source":"def get_raw_txt(_id, path=\"train\"):\n    _d = json.loads(open(f\"{PATH}/{path}/{_id}.json\").read())\n    return \" \".join([i[\"text\"] for i in _d])\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef clean_text2(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\nkeywords = set([\"data\", \"study\", \"survey\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the custom model and predict.","metadata":{}},{"cell_type":"code","source":"import spacy\nfrom thinc.api import set_gpu_allocator, require_gpu\nset_gpu_allocator(\"pytorch\")\nrequire_gpu(0)\nnlp = spacy.load(\"../input/spacycustomnerbest/model-best\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# from spacy import displacy\nPATH = \"../input/coleridgeinitiative-show-us-the-data/\"\ntqdm_params = dict(bar_format='{desc}{bar} [ {n} / {total} (remaining: {remaining}) ]', colour=\"darkgreen\")\n\ntqdm.pandas(desc=\"Text download from JSON files status :\",**tqdm_params)\nsubmission = pd.read_csv(f\"{PATH}sample_submission.csv\")\nsubmission['text'] = submission['Id'].progress_apply(partial(get_raw_txt,path=\"test\"))\n\ntqdm.pandas(desc=\"Text processing status :\",**tqdm_params)\nfor i in tqdm(submission.index, total=submission.shape[0]):\n    sentences = submission.loc[i]['text'].split('.')\n    predictions = set()\n    for sentence in sentences:\n        if len(sentence)>20:\n            sentence = clean_text2(sentence)\n            if any(kw in sentence.lower() for kw in keywords):\n                doc = nlp(sentence)\n#                 displacy.render(doc, style=\"ent\", jupyter=True)\n                if len(doc.ents) > 0:\n                    for ent in doc.ents:\n                        predictions.add(clean_text(ent.text))\n    submission.loc[i,\"PredictionString\"] = '|'.join(list(predictions))\n\nsubmission = submission.drop(\"text\",axis=1)\nsubmission.to_csv(\"submission.csv\", index = False)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}