{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nThis notebook is a continuation of my [EDA Notebook](https://www.kaggle.com/jagdmir/coleridge-ner-using-spacy)\nI have tried to use SPACY model(NER) to identify the datasets within the publications!","metadata":{}},{"cell_type":"markdown","source":"Named-entity recognition (NER) is the process of automatically identifying the entities discussed in a text and classifying them into pre-defined categories such as ‘person’, ‘organization’, ‘location’ and so on. \n\nThe spaCy library allows you to train NER models by both updating an existing spacy model to suit the specific context of your text documents and also to train a fresh NER model from scratch.\n\nNamed Entity Recognition is implemented by the pipeline component `ner`. Most of the models have it in their processing pipeline by default.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os,re\nimport glob\nimport json\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-06T14:02:25.849380Z","iopub.execute_input":"2021-06-06T14:02:25.849897Z","iopub.status.idle":"2021-06-06T14:02:25.860667Z","shell.execute_reply.started":"2021-06-06T14:02:25.849813Z","shell.execute_reply":"2021-06-06T14:02:25.859779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load train.csv\ntrain_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:25.862223Z","iopub.execute_input":"2021-06-06T14:02:25.862614Z","iopub.status.idle":"2021-06-06T14:02:26.008384Z","shell.execute_reply.started":"2021-06-06T14:02:25.862554Z","shell.execute_reply":"2021-06-06T14:02:26.007603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no. of unique labels in the dataset\ntrain_csv.dataset_label.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:26.010105Z","iopub.execute_input":"2021-06-06T14:02:26.010446Z","iopub.status.idle":"2021-06-06T14:02:26.018872Z","shell.execute_reply.started":"2021-06-06T14:02:26.010410Z","shell.execute_reply":"2021-06-06T14:02:26.017865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take one sample of each of the dataset label\ntrain_csv.drop_duplicates(subset=\"dataset_label\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:26.020779Z","iopub.execute_input":"2021-06-06T14:02:26.021201Z","iopub.status.idle":"2021-06-06T14:02:26.031308Z","shell.execute_reply.started":"2021-06-06T14:02:26.021167Z","shell.execute_reply":"2021-06-06T14:02:26.030496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a copy to the training dataset\ntrain = train_csv.copy()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:26.032517Z","iopub.execute_input":"2021-06-06T14:02:26.033146Z","iopub.status.idle":"2021-06-06T14:02:26.039560Z","shell.execute_reply.started":"2021-06-06T14:02:26.033108Z","shell.execute_reply":"2021-06-06T14:02:26.038730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation\n\nSpaCy accepts training data as list of tuples.\n\nEach tuple should contain the text and a dictionary. \n\nThe dictionary should hold the `start` and `end` indices of the `named enity` in the text, and the `category or label` of the named entity.\n\nFor example, (\"Walmart is a leading e-commerce company\", {\"entities\": [(0, 7, \"ORG\")]})\n\nLet's do this!","metadata":{}},{"cell_type":"code","source":"def clean_sentence(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())   ","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:26.040892Z","iopub.execute_input":"2021-06-06T14:02:26.041262Z","iopub.status.idle":"2021-06-06T14:02:26.048809Z","shell.execute_reply.started":"2021-06-06T14:02:26.041229Z","shell.execute_reply":"2021-06-06T14:02:26.047991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nDATA = []\nent_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n    \n    balanced = False\n\n    sentences = nltk.tokenize.sent_tokenize(str(data))\n\n    for sentence in sentences:          \n        sentence = clean_sentence(sentence).strip()        \n        a = re.search(row.dataset_label.lower(),sentence)            \n        if  a != None:\n            DATA.append((sentence,{\"entities\":[(a.span()[0],a.span()[1],\"DATASET\")]}))\n            ent_count = ent_count + 1\n            balanced = True\n        else:\n            if balanced:\n                DATA.append((sentence,{\"entities\":[]}))\n                empty_count = empty_count + 1\n                balanced = False\nprint(\"Text with entities:\",ent_count,\"Text without entities:\",empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:02:26.051745Z","iopub.execute_input":"2021-06-06T14:02:26.051996Z","iopub.status.idle":"2021-06-06T14:03:10.436398Z","shell.execute_reply.started":"2021-06-06T14:02:26.051974Z","shell.execute_reply":"2021-06-06T14:03:10.435516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(DATA)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:03:10.438684Z","iopub.execute_input":"2021-06-06T14:03:10.439047Z","iopub.status.idle":"2021-06-06T14:03:10.444337Z","shell.execute_reply.started":"2021-06-06T14:03:10.438995Z","shell.execute_reply":"2021-06-06T14:03:10.443348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATA = DATA","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:03:10.446157Z","iopub.execute_input":"2021-06-06T14:03:10.446863Z","iopub.status.idle":"2021-06-06T14:03:10.454372Z","shell.execute_reply.started":"2021-06-06T14:03:10.446824Z","shell.execute_reply":"2021-06-06T14:03:10.453464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Building\n\n1. To train an ner model, the model has to be looped over the example for sufficient number of iterations. \n\n2. Before every iteration it’s a good practice to shuffle the examples randomly throughrandom.shuffle() function .\n   This will ensure the model does not make generalizations based on the order of the examples.\n\n3. The training data is usually passed in batches. \n   We can call the minibatch() function of spaCy over the training data that will return you data in batches . \n   The minibatch function takes size parameter to denote the batch size. \n   \n4. In each iteration , the model or ner is updated through the nlp.update() command. \n\n   Parameters of nlp.update() are :\n\n*     docs: This expects a batch of texts as input. You can pass each batch to the zip method, which will return you batches     of text and annotations.\n \n*     golds: You can pass the annotations we got through zip method here\n \n*     drop: This represents the dropout rate.\n \n*     losses: A dictionary to hold the losses against each pipeline component. Create an empty dictionary and pass it here.\n \nAt each word, the update() it makes a prediction. It then consults the annotations to check if the prediction is right. If it isn’t , it adjusts the weights so that the correct action will score higher next time.\n\nFinally, all of the training is done within the context of the nlp model with disabled pipeline, to prevent the other components from being involved.","metadata":{}},{"cell_type":"code","source":"import random\nimport spacy\nfrom spacy.util import minibatch, compounding\nfrom pathlib import Path\n\ndef train_spacy(TRAIN_DATA, iterations, model):\n    #TRAIN_DATA = data\n    print(f\"downloads = {model}\")\n    if model is not None:\n        print(f\"training existing model\")\n        nlp = spacy.load(model)\n        print(\"Model is Loaded '%s'\" % model)\n    else:\n        print(f\"Creating new model\")\n\n        nlp = spacy.blank('en')  # create blank Language class\n\n    if 'ner' not in nlp.pipe_names:\n        ner = nlp.create_pipe('ner')\n        nlp.add_pipe(ner, last=True)\n    else:\n        ner = nlp.get_pipe('ner')\n\n    # Based on template, get labels and save those for further training\n    \n    for _, annotations in TRAIN_DATA:\n        for ent in annotations.get(\"entities\"):\n            ner.add_label(ent[2])\n\n    # get names of other pipes to disable them during training\n    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']\n    with nlp.disable_pipes(*other_pipes):  # only train NER\n        if model is None:\n            optimizer = nlp.begin_training()\n        else:\n            optimizer = nlp.entity.create_optimizer()\n        tags = dict()\n        for itn in range(iterations):\n            print(\"Starting iteration \" + str(itn))\n            random.shuffle(TRAIN_DATA)\n            losses = {}\n            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 16.0, 1.001))\n            # type 2 with mini batch\n            for batch in batches:                \n                texts, annotations = zip(*batch)                \n                golds = annotations \n                nlp.update(\n                    texts,  # batch of texts\n                    golds,  # batch of annotations\n                    drop=0.2,  # dropout - make it harder to memorise data                    \n                    losses=losses,\n                    sgd=optimizer\n                )\n            print(losses)\n    return nlp","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:03:10.455724Z","iopub.execute_input":"2021-06-06T14:03:10.456099Z","iopub.status.idle":"2021-06-06T14:03:12.149182Z","shell.execute_reply.started":"2021-06-06T14:03:10.456066Z","shell.execute_reply":"2021-06-06T14:03:12.148298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model for 1 iteration (for faster submission)\nmodel = train_spacy(TRAIN_DATA,10,\"en_core_web_sm\") # pass \"en_core_web_sm\" if you want to use pre trained spacy model","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:03:12.150373Z","iopub.execute_input":"2021-06-06T14:03:12.150710Z","iopub.status.idle":"2021-06-06T14:30:12.172067Z","shell.execute_reply.started":"2021-06-06T14:03:12.150676Z","shell.execute_reply":"2021-06-06T14:30:12.171023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Predictions","metadata":{}},{"cell_type":"code","source":"# getting list of publication ids in the test set\ntest_pubs = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\").Id\ntest_pubs\n\n# load submission.csv\nsub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:30:12.173329Z","iopub.execute_input":"2021-06-06T14:30:12.173796Z","iopub.status.idle":"2021-06-06T14:30:12.188135Z","shell.execute_reply.started":"2021-06-06T14:30:12.173757Z","shell.execute_reply":"2021-06-06T14:30:12.187341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\n\nfor pub in test_pubs:    \n    print(\"pub:\",pub)\n    \n    f = open(\"../input/coleridgeinitiative-show-us-the-data/test/\" + pub + \".json\")  \n    \n    predicted_text = \"\"\n    \n    data = json.load(f)      \n\n    sentences = nltk.tokenize.sent_tokenize(str(data))\n\n    for sentence in sentences:          \n        sentence = clean_sentence(sentence).strip()        \n        doc = model(sentence)\n        for ent in doc.ents:\n            predicted_text = predicted_text + ent.text + \"|\"\n    \n            #print(\"pub:\",pub, \"\\n\",predicted_text[:-1].strip(),\"\\n\")\n\n    print(\"final:\",predicted_text[:-1])\n    sub.PredictionString.loc[i] = predicted_text[:-1].strip()\n    \n    i = i + 1","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:39:44.570758Z","iopub.execute_input":"2021-06-06T14:39:44.571176Z","iopub.status.idle":"2021-06-06T14:40:00.129698Z","shell.execute_reply.started":"2021-06-06T14:39:44.571140Z","shell.execute_reply":"2021-06-06T14:40:00.128240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finally!\nsub.to_csv('submission.csv',index=False)\nsub\n","metadata":{"execution":{"iopub.status.busy":"2021-06-06T14:30:28.268197Z","iopub.execute_input":"2021-06-06T14:30:28.268445Z","iopub.status.idle":"2021-06-06T14:30:28.402670Z","shell.execute_reply.started":"2021-06-06T14:30:28.268419Z","shell.execute_reply":"2021-06-06T14:30:28.401928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}