{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":25925,"databundleVersionId":2089362,"sourceType":"competition"},{"sourceId":1496462,"sourceType":"datasetVersion","datasetId":878950},{"sourceId":2167021,"sourceType":"datasetVersion","datasetId":1300788},{"sourceId":2356912,"sourceType":"datasetVersion","datasetId":1423275}],"dockerImageVersionId":30096,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T16:17:41.697855Z","iopub.execute_input":"2021-06-22T16:17:41.698245Z","iopub.status.idle":"2021-06-22T16:17:49.903356Z","shell.execute_reply.started":"2021-06-22T16:17:41.698167Z","shell.execute_reply":"2021-06-22T16:17:49.902251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall fastai en-core-web-sm en-core-web-lg spacy -y -q","metadata":{"execution":{"iopub.status.busy":"2021-08-10T10:31:35.662867Z","iopub.execute_input":"2021-08-10T10:31:35.663603Z","iopub.status.idle":"2021-08-10T10:31:43.689685Z","shell.execute_reply.started":"2021-08-10T10:31:35.663469Z","shell.execute_reply":"2021-08-10T10:31:43.688727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/spacy3/catalogue-2.0.3-py3-none-any.whl ../input/spacy3/typer-0.3.2-py3-none-any.whl ../input/spacy3/srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/pathy-0.5.2-py3-none-any.whl ../input/spacy3/smart_open-3.0.0-py3-none-any.whl ../input/spacy3/pydantic-1.7.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/thinc-8.0.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy-3.0.6-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_legacy-3.0.5-py2.py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:19:48.600978Z","iopub.execute_input":"2021-06-22T16:19:48.601325Z","iopub.status.idle":"2021-06-22T16:20:14.326281Z","shell.execute_reply.started":"2021-06-22T16:19:48.601287Z","shell.execute_reply":"2021-06-22T16:20:14.325298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/spacy3/en_core_web_lg-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_md-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_sm-3.0.0-py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:21:22.279547Z","iopub.execute_input":"2021-06-22T16:21:22.279906Z","iopub.status.idle":"2021-06-22T16:21:46.137829Z","shell.execute_reply.started":"2021-06-22T16:21:22.279876Z","shell.execute_reply":"2021-06-22T16:21:46.136585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/spacy3/spacy_alignments-0.8.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_transformers-1.0.2-py2.py3-none-any.whl ../input/spacy3/en_core_web_trf-3.0.0-py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:21:46.142461Z","iopub.execute_input":"2021-06-22T16:21:46.142811Z","iopub.status.idle":"2021-06-22T16:22:03.154422Z","shell.execute_reply.started":"2021-06-22T16:21:46.142781Z","shell.execute_reply":"2021-06-22T16:22:03.153159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nassert spacy.__version__ == '3.0.6'","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:22:03.156293Z","iopub.execute_input":"2021-06-22T16:22:03.156727Z","iopub.status.idle":"2021-06-22T16:22:12.985849Z","shell.execute_reply.started":"2021-06-22T16:22:03.156684Z","shell.execute_reply":"2021-06-22T16:22:12.984735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport json\nimport glob\nimport re\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:35:25.479985Z","iopub.execute_input":"2021-06-22T16:35:25.480628Z","iopub.status.idle":"2021-06-22T16:35:25.485951Z","shell.execute_reply.started":"2021-06-22T16:35:25.480557Z","shell.execute_reply":"2021-06-22T16:35:25.485126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandarallel import pandarallel\npandarallel.initialize()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:23:09.026228Z","iopub.execute_input":"2021-06-22T16:23:09.026583Z","iopub.status.idle":"2021-06-22T16:23:09.044483Z","shell.execute_reply.started":"2021-06-22T16:23:09.026551Z","shell.execute_reply":"2021-06-22T16:23:09.043255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"custom_ner_model = spacy.load(\"../input/original-model/spaCy/output/model-best\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:35:00.378234Z","iopub.execute_input":"2021-06-22T16:35:00.378841Z","iopub.status.idle":"2021-06-22T16:35:11.651374Z","shell.execute_reply.started":"2021-06-22T16:35:00.378804Z","shell.execute_reply":"2021-06-22T16:35:11.650244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:35:32.611928Z","iopub.execute_input":"2021-06-22T16:35:32.612289Z","iopub.status.idle":"2021-06-22T16:35:32.635328Z","shell.execute_reply.started":"2021-06-22T16:35:32.612261Z","shell.execute_reply":"2021-06-22T16:35:32.634423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = '../input/coleridgeinitiative-show-us-the-data/test'\n\npapers = {}\nfor paper_id in submission_df['Id']:\n    with open(f'{test_files}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:37:00.927989Z","iopub.execute_input":"2021-06-22T16:37:00.928408Z","iopub.status.idle":"2021-06-22T16:37:00.993834Z","shell.execute_reply.started":"2021-06-22T16:37:00.928375Z","shell.execute_reply":"2021-06-22T16:37:00.992654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef jaccard(str1, str2): \n    \"\"\"\n    defined by the competition\n    \"\"\"\n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:37:05.142301Z","iopub.execute_input":"2021-06-22T16:37:05.142702Z","iopub.status.idle":"2021-06-22T16:37:05.149337Z","shell.execute_reply.started":"2021-06-22T16:37:05.142666Z","shell.execute_reply":"2021-06-22T16:37:05.148001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ner_apply(file):\n    candidates = []\n\n    for paper_id in file['Id']:\n        paper = papers[paper_id]\n        paper_text = '. '.join(section['text'] for section in paper)\n        sentences = paper_text.split(\".\")\n#     sentences = sent_tokenize(paper_text)\n        label = set()\n        for sentence in sentences:\n#         sentence = clean_text(sentence)\n            doc = custom_ner_model(sentence)\n            if len(doc.ents) > 0:\n#             displacy.render(doc, style=\"ent\", jupyter=True)\n                label.add(clean_text(doc.ents))\n#    sorted(candidates)\n        label_list = list(label)\n        sorted(label_list)\n        candidates.append('|'.join(label_list))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:40:21.505773Z","iopub.execute_input":"2021-06-22T16:40:21.506164Z","iopub.status.idle":"2021-06-22T16:40:21.513801Z","shell.execute_reply.started":"2021-06-22T16:40:21.506133Z","shell.execute_reply":"2021-06-22T16:40:21.512687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:54:07.235574Z","iopub.execute_input":"2021-06-22T16:54:07.236198Z","iopub.status.idle":"2021-06-22T16:54:07.341243Z","shell.execute_reply.started":"2021-06-22T16:54:07.236147Z","shell.execute_reply":"2021-06-22T16:54:07.340075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nres = train_csv.apply(clean_text, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:56:42.818605Z","iopub.execute_input":"2021-06-22T16:56:42.819001Z","iopub.status.idle":"2021-06-22T16:56:53.141375Z","shell.execute_reply.started":"2021-06-22T16:56:42.818967Z","shell.execute_reply":"2021-06-22T16:56:53.140097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nres_parallel = train_csv.parallel_apply(clean_text, axis=1)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:57:13.114123Z","iopub.execute_input":"2021-06-22T16:57:13.114511Z","iopub.status.idle":"2021-06-22T16:57:17.393766Z","shell.execute_reply.started":"2021-06-22T16:57:13.114478Z","shell.execute_reply":"2021-06-22T16:57:17.392799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.equals(res_parallel)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T16:57:21.014042Z","iopub.execute_input":"2021-06-22T16:57:21.01446Z","iopub.status.idle":"2021-06-22T16:57:21.026534Z","shell.execute_reply.started":"2021-06-22T16:57:21.014424Z","shell.execute_reply":"2021-06-22T16:57:21.025402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"./submission.csv\",index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}