{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-23T05:27:29.610201Z","iopub.execute_input":"2021-07-23T05:27:29.610621Z","iopub.status.idle":"2021-07-23T05:27:56.962830Z","shell.execute_reply.started":"2021-07-23T05:27:29.610534Z","shell.execute_reply":"2021-07-23T05:27:56.961506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport glob\nimport json\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:30:06.203430Z","iopub.execute_input":"2021-07-23T05:30:06.203924Z","iopub.status.idle":"2021-07-23T05:30:06.209812Z","shell.execute_reply.started":"2021-07-23T05:30:06.203885Z","shell.execute_reply":"2021-07-23T05:30:06.208568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:30:21.891864Z","iopub.execute_input":"2021-07-23T05:30:21.892250Z","iopub.status.idle":"2021-07-23T05:30:22.097214Z","shell.execute_reply.started":"2021-07-23T05:30:21.892215Z","shell.execute_reply":"2021-07-23T05:30:22.095899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.drop_duplicates(subset=\"dataset_label\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:30:51.936590Z","iopub.execute_input":"2021-07-23T05:30:51.937046Z","iopub.status.idle":"2021-07-23T05:30:51.952117Z","shell.execute_reply.started":"2021-07-23T05:30:51.937006Z","shell.execute_reply":"2021-07-23T05:30:51.951012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_csv.copy()","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:31:01.656765Z","iopub.execute_input":"2021-07-23T05:31:01.657172Z","iopub.status.idle":"2021-07-23T05:31:01.663263Z","shell.execute_reply.started":"2021-07-23T05:31:01.657130Z","shell.execute_reply":"2021-07-23T05:31:01.662015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_sentence(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())   ","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:31:26.185126Z","iopub.execute_input":"2021-07-23T05:31:26.185580Z","iopub.status.idle":"2021-07-23T05:31:26.191318Z","shell.execute_reply.started":"2021-07-23T05:31:26.185545Z","shell.execute_reply":"2021-07-23T05:31:26.190186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nDATA = []\nent_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n    \n    balanced = False\n\n    sentences = nltk.tokenize.sent_tokenize(str(data))\n\n    for sentence in sentences:          \n        sentence = clean_sentence(sentence).strip()        \n        a = re.search(row.dataset_label.lower(),sentence)            \n        if  a != None:\n            DATA.append((sentence,{\"entities\":[(a.span()[0],a.span()[1],\"DATASET\")]}))\n            ent_count = ent_count + 1\n            balanced = True\n        else:\n            if balanced:\n                DATA.append((sentence,{\"entities\":[]}))\n                empty_count = empty_count + 1\n                balanced = False\nprint(\"Text with entities:\",ent_count,\"Text without entities:\",empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:31:48.437376Z","iopub.execute_input":"2021-07-23T05:31:48.437834Z","iopub.status.idle":"2021-07-23T05:32:44.167024Z","shell.execute_reply.started":"2021-07-23T05:31:48.437784Z","shell.execute_reply":"2021-07-23T05:32:44.166066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(DATA)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:33:00.868185Z","iopub.execute_input":"2021-07-23T05:33:00.868543Z","iopub.status.idle":"2021-07-23T05:33:00.874326Z","shell.execute_reply.started":"2021-07-23T05:33:00.868511Z","shell.execute_reply":"2021-07-23T05:33:00.873162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATA = DATA","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:33:15.095232Z","iopub.execute_input":"2021-07-23T05:33:15.095829Z","iopub.status.idle":"2021-07-23T05:33:15.100322Z","shell.execute_reply.started":"2021-07-23T05:33:15.095769Z","shell.execute_reply":"2021-07-23T05:33:15.099216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport spacy\nfrom spacy.util import minibatch, compounding\nfrom pathlib import Path\n\ndef train_model_spacy(TRAIN_DATA, iterations, model):\n    print(f\"downloads = {model}\")\n    if model is not None:\n        print(\"Model is existing!\")\n        nlp=spacy.load(model)\n        print(\"Model is loaded\")\n    else:\n        print(\"Creating new model\")\n        nlp=spacy.blank('en')\n    if 'ner' not in nlp.pipe_names:\n        ner = nlp.create_pipe('ner')\n        nlp.add_pipe(ner, last=True)\n    else:\n        ner = nlp.get_pipe('ner')\n    \n    for _, annotations in TRAIN_DATA:\n        for ent in annotations.get(\"entities\"):\n            ner.add_label(ent[2])\n    \n    #disable other pipes\n    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']\n    with nlp.disable_pipes(*other_pipes):  # only train NER\n        if model is None:\n            optimizer = nlp.begin_training()\n        else:\n            optimizer = nlp.entity.create_optimizer()\n        tags = dict()\n        for itn in range(iterations):\n            print(\"Starting iteration \" + str(itn))\n            random.shuffle(TRAIN_DATA)\n            losses = {}\n            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 16.0, 1.001))\n            # type 2 with mini batch\n            for batch in batches:                \n                texts, annotations = zip(*batch)                \n                golds = annotations \n                nlp.update(\n                    texts,  # batch of texts\n                    golds,  # batch of annotations\n                    drop=0.2,  # dropout - make it harder to memorise data                    \n                    losses=losses,\n                    sgd=optimizer\n                )\n            print(losses)\n    return nlp","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:42:05.177966Z","iopub.execute_input":"2021-07-23T05:42:05.178375Z","iopub.status.idle":"2021-07-23T05:42:05.970268Z","shell.execute_reply.started":"2021-07-23T05:42:05.178341Z","shell.execute_reply":"2021-07-23T05:42:05.969123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_model_spacy(TRAIN_DATA,15,\"en_core_web_sm\")","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:43:03.055253Z","iopub.execute_input":"2021-07-23T05:43:03.055726Z","iopub.status.idle":"2021-07-23T05:45:24.936759Z","shell.execute_reply.started":"2021-07-23T05:43:03.055660Z","shell.execute_reply":"2021-07-23T05:45:24.935598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pubs = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\").Id\ntest_pubs\n\n# load submission.csv\nsub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:46:30.235645Z","iopub.execute_input":"2021-07-23T05:46:30.236096Z","iopub.status.idle":"2021-07-23T05:46:30.260777Z","shell.execute_reply.started":"2021-07-23T05:46:30.236058Z","shell.execute_reply":"2021-07-23T05:46:30.259773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\n\nfor pub in test_pubs:    \n    print(\"pub:\",pub)\n    \n    f = open(\"../input/coleridgeinitiative-show-us-the-data/test/\" + pub + \".json\")  \n    \n    predicted_text = \"\"\n    \n    data = json.load(f)      \n\n    sentences = nltk.tokenize.sent_tokenize(str(data))\n\n    for sentence in sentences:          \n        sentence = clean_sentence(sentence).strip()        \n        doc = model(sentence)\n        for ent in doc.ents:\n            predicted_text = predicted_text + ent.text + \"|\"\n    \n            #print(\"pub:\",pub, \"\\n\",predicted_text[:-1].strip(),\"\\n\")\n\n    print(\"final:\",predicted_text[:-1])\n    sub.PredictionString.loc[i] = predicted_text[:-1].strip()\n    \n    i = i + 1","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:46:59.538765Z","iopub.execute_input":"2021-07-23T05:46:59.539126Z","iopub.status.idle":"2021-07-23T05:47:22.768155Z","shell.execute_reply.started":"2021-07-23T05:46:59.539091Z","shell.execute_reply":"2021-07-23T05:47:22.766917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv',index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2021-07-23T05:47:42.755873Z","iopub.execute_input":"2021-07-23T05:47:42.756281Z","iopub.status.idle":"2021-07-23T05:47:42.771658Z","shell.execute_reply.started":"2021-07-23T05:47:42.756247Z","shell.execute_reply":"2021-07-23T05:47:42.770860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}