{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Custom Ner Template\n\nSubstitute the spacy part with NLTK for example","metadata":{}},{"cell_type":"code","source":"import json\nimport glob\nimport re\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\n\nimport spacy\nfrom spacy.util import minibatch, compounding\nfrom spacy import warnings\nspacy.prefer_gpu()\n\nimport nltk\nfrom nltk import sent_tokenize\n\nSTRINGMATCHING = False","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:22:39.496805Z","iopub.execute_input":"2021-05-27T18:22:39.497191Z","iopub.status.idle":"2021-05-27T18:22:40.374671Z","shell.execute_reply.started":"2021-05-27T18:22:39.497145Z","shell.execute_reply":"2021-05-27T18:22:40.373593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the files into the program\n","metadata":{}},{"cell_type":"code","source":"# Has Id, pub_title, dataset_title, dataset_label, cleaned_label\ntrain = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain.info()\n\ntrain_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:22:40.376095Z","iopub.execute_input":"2021-05-27T18:22:40.376422Z","iopub.status.idle":"2021-05-27T18:22:40.535133Z","shell.execute_reply.started":"2021-05-27T18:22:40.376392Z","shell.execute_reply":"2021-05-27T18:22:40.534163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_publications = pd.DataFrame()\ndf_test_publications = pd.DataFrame()\n\nfor train_file in tqdm(train_files):\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id',train_file.split('/')[-1].split('.')[0])\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications.to_csv(\"df_train_publications.csv\",index=False)\n\nfor test_file in tqdm(test_files):\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)\n\nsubmission_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\",index_col = 0)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:22:40.536976Z","iopub.execute_input":"2021-05-27T18:22:40.537424Z","iopub.status.idle":"2021-05-27T18:27:26.9896Z","shell.execute_reply.started":"2021-05-27T18:22:40.53738Z","shell.execute_reply":"2021-05-27T18:27:26.988524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing\n\nCutting the strings at dot-space","metadata":{}},{"cell_type":"code","source":"train_data = []\np_noentity = 0.1 # also use sentences with no entities for training\n\ntrain_files = train_files[:100]\n\nfor file in tqdm(train_files):\n    with open(file) as f:\n        data = json.load(f)\n        f_id = file.split('/')[-1].split('.')[0] # get the publication id\n        # get the dataset titles of the publication ID\n        f_datasettitles = [x.lower() for x in set(train[train[\"Id\"]==f_id][\"dataset_title\"]).union(set(train[train[\"Id\"]==f_id][\"dataset_label\"].unique()))]\n        \n        for paragraph in data:\n            text = paragraph['text']\n            sentences = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', text) # split it according to some nice regular expression https://stackoverflow.com/questions/25735644/python-regex-for-splitting-text-into-sentences-sentence-tokenizing\n            \n            for s in sentences:\n                s = re.sub('[^A-Za-z0-9]+', ' ', s) # remove any special characters that are not letters or numbers\n                s = s.lower()\n                if len(s) > 5: # check whether the sentence has a minimal amount of characters\n                    for datasettitle in f_datasettitles:\n                        datasettitle = datasettitle.lower()\n                        if datasettitle in s:\n                            start = s.find(datasettitle)\n                            end = start + len(datasettitle)\n                            train_data.append((s,{\"entities\": [(start,end, \"DATAS\")]})) # append a tupel of readable by spacy\n                        elif np.random.rand() < p_noentity:\n                            train_data.append((s,{\"entities\": []})) # append the sentence with no entities\n\nnp.save(\"/kaggle/working/annotated.npy\",np.array(train_data))\nprint(len(train_data[0]))","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:27:26.991329Z","iopub.execute_input":"2021-05-27T18:27:26.991914Z","iopub.status.idle":"2021-05-27T18:27:28.812341Z","shell.execute_reply.started":"2021-05-27T18:27:26.991868Z","shell.execute_reply":"2021-05-27T18:27:28.811265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(train_data)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:27:28.813834Z","iopub.execute_input":"2021-05-27T18:27:28.814167Z","iopub.status.idle":"2021-05-27T18:27:28.818455Z","shell.execute_reply.started":"2021-05-27T18:27:28.814115Z","shell.execute_reply":"2021-05-27T18:27:28.817289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NLTK Model\n","metadata":{}},{"cell_type":"markdown","source":"<!-- ## Train Spacy NER model -->\n","metadata":{}},{"cell_type":"code","source":"# print(len(train_data))\n\nLABEL = \"DATAS\"\n\nspacy.prefer_gpu()\nimport en_core_web_sm\nnlp = en_core_web_sm.load()\n\nnlp = spacy.load(\"en_core_web_sm\")\nner = nlp.get_pipe('ner')\nner.add_label(LABEL)\n\noptimizer = nlp.resume_training()\nmove_names = list(ner.move_names)\n\npipe_exceptions = [\"ner\",\"trf_wordpiecer\",\"trf_tok2vec\"]\nother_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]\n\nloss_list = []\nloss_list.append(len(train_data))\n\nwith nlp.disable_pipes(*other_pipes), warnings.catch_warnings():\n    warnings.filterwarnings(\"ignore\", category=UserWarning, module='spacy') # some entities are misaligned due to additional numbers in adni1, we just ignore this for now\n    sizes = compounding(1.0, 4.0, 1.001)\n    # Training for 30 iterations     \n    for itn in tqdm(range(5)):\n        # shuffle examples before training\n        random.shuffle(train_data)\n        # batch up the examples using spaCy's minibatch\n        batches = minibatch(train_data, size=sizes)\n        # ictionary to store losses\n        losses = {}\n        for batch in batches:\n            texts, annotations = zip(*batch)\n            if len(annotations[0][\"entities\"]) > 0 and False: # show if the warnings occure\n                a = annotations[0][\"entities\"][0]\n                print(a[0])\n                print(a[1])\n                print(texts[0])\n                print(texts[0][a[0]:a[1]])\n                print(spacy.gold.biluo_tags_from_offsets(nlp.make_doc(texts[0]),annotations[0]['entities']))\n                \n            # Calling update() over the iteration\n            nlp.update(texts, annotations, sgd=optimizer, drop=0.35, losses=losses)\n        print(\"Losses\", losses)\n        print(losses[\"ner\"])\n        loss_list.append(losses[\"ner\"])\nnp.save(\"/kaggle/working/losses.npy\",np.array(loss_list))","metadata":{"execution":{"iopub.status.busy":"2021-05-27T18:27:28.820135Z","iopub.execute_input":"2021-05-27T18:27:28.82057Z","iopub.status.idle":"2021-05-27T18:31:41.187021Z","shell.execute_reply.started":"2021-05-27T18:27:28.820528Z","shell.execute_reply":"2021-05-27T18:31:41.185887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Spacy NER model","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\npred_labels = []\n\nfor index in submission_df.index:\n    try:\n        publication_text = df_test_publications[df_test_publications['pub_id']==index].text.str.cat(sep=\"\\n\").lower() # concatenate all paragraphs\n        sentences = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', publication_text) # split at the sentences\n\n        pred_label = []\n        for s in sentences:\n            s = re.sub('[^A-Za-z0-9]+', ' ', s) # remove any special characters that are not letters or numbers\n            s = s.lower()\n            doc = nlp(s)\n            if len(doc.ents)>=1:\n                for ent in doc.ents:\n                    pred_label.append(clean_text(str(ent)))\n\n        if len(pred_label) is 0:\n            pred_label.append(clean_text(\"empty\"))\n    except:\n        pred_label.append(\"error\")\n    \n    pred_labels.append(\"|\".join(pred_label))\n\n\nsubmission_df['PredictionString'] = pred_labels\n\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:38:24.707977Z","iopub.execute_input":"2021-05-25T15:38:24.708331Z","iopub.status.idle":"2021-05-25T15:38:33.449212Z","shell.execute_reply.started":"2021-05-25T15:38:24.708289Z","shell.execute_reply":"2021-05-25T15:38:33.448329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## \"Train and test\" Stringmatching","metadata":{}},{"cell_type":"code","source":"# String matching\nif STRINGMATCHING:\n    def clean_text(txt):\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n    # get the unique dataset titles\n    datasets_titles = [x.lower() for x in set(train['dataset_title'].unique()).union(set(train['dataset_label'].unique()))]\n    labels = []\n\n    for index in submission_df.index:\n        # get the text of all the sections and concatenate in new line\n        publication_text = df_test_publications[df_test_publications['pub_id']==index].text.str.cat(sep=\"\\n\").lower()\n\n        label = []\n        for dataset_title in datasets_titles:\n            if dataset_title in publication_text:\n                label.append(clean_text(dataset_title))\n        labels.append('|'.join(label))\n    print(labels)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:38:33.450559Z","iopub.execute_input":"2021-05-25T15:38:33.450848Z","iopub.status.idle":"2021-05-25T15:38:33.457763Z","shell.execute_reply.started":"2021-05-25T15:38:33.450821Z","shell.execute_reply":"2021-05-25T15:38:33.456762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating Testset","metadata":{}},{"cell_type":"code","source":"if STRINGMATCHING:\n    submission_df['PredictionString'] = labels\n\n    submission_df.to_csv('submission.csv')\n\n    submission_df","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:38:33.459976Z","iopub.execute_input":"2021-05-25T15:38:33.460281Z","iopub.status.idle":"2021-05-25T15:38:33.47198Z","shell.execute_reply.started":"2021-05-25T15:38:33.460251Z","shell.execute_reply":"2021-05-25T15:38:33.471147Z"},"trusted":true},"execution_count":null,"outputs":[]}]}