{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Introduction"},{"metadata":{},"cell_type":"markdown","source":"Hello everyone! This notebook is an attempt to look at the problem like on a NER-problem (Named Entity Recognition). For this purpose I have used the spacy library. You are welcome to write any comments and suggestions to imrove the result!"},{"metadata":{},"cell_type":"markdown","source":"## Load the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\nimport glob\ntrain_files = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/*.json\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate dataframes from jsons"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\ndf_train_publications = pd.DataFrame()\n\nfor train_file in tqdm(train_files):\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0].replace('train\\\\', ''))\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_publications = pd.DataFrame()\n\nfor test_file in tqdm(test_files):\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0].replace('test\\\\', ''))\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implement several functions to concatenate text"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compare(column):\n    return '|'.join(list(set(column)))\n\ndef make_list(df):\n    ids = df['Id'].unique()\n    df1 = pd.DataFrame(columns=['Id', 'cleaned_label'])\n    for id_ in ids:\n        df1 = pd.concat([df1, pd.DataFrame({\"Id\": id_, \"cleaned_label\":[df[df['Id']==id_].apply(compare)['cleaned_label']]})])\n    return df1.drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def concat(column):\n    res = ' '\n    for st in column:\n        if type(st) == str:\n            res += st\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = df_train_publications.groupby('pub_id')['text'].apply(concat).reset_index()\n\ntrain.loc[train['pub_id'].isin(train_csv['Id']), 'cleaned_label'] = train_csv.loc[train_csv['Id'].isin(train['pub_id']),'cleaned_label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = df_test_publications.groupby('pub_id')['text'].apply(concat).reset_index()\ntest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Named Entity Recognition"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NER\nimport spacy\nimport random\nfrom spacy.util import minibatch, compounding\nfrom pathlib import Path\n\nfrom spacy.util import minibatch, compounding\nfrom pathlib import Path\nimport random\n\ndef chunks(lst, n):\n    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n    for i in range(0, len(lst), n):\n        yield lst[i:i + n]\n\ndef train_spacy(data,iterations):\n    TRAIN_DATA = [] \n    for i in range(300):\n        TRAIN_DATA.append(random.choice(data))\n\n    nlp=spacy.blank('en', disable=['parser'])\n\n    if 'ner' not in nlp.pipe_names:\n        ner = nlp.create_pipe('ner')\n        nlp.add_pipe(ner)\n    else:\n        ner = nlp.get_pipe('ner')\n        \n    ner.add_label('DATASET')\n\n    optimizer = nlp.begin_training()\n    for itn in range(iterations):\n        print(\"Statring iteration \" + str(itn))\n        random.shuffle(TRAIN_DATA)\n        losses = {}\n        for text, annotations in tqdm(TRAIN_DATA):\n            nlp.update(\n                [text],\n                [annotations],\n                drop=0.2,\n                sgd=optimizer,\n                losses=losses)\n        print(\"ITERATION {}, Losses\".format(itn), losses)\n    return nlp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make TRAIN_DATA"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DATA = []\nfor idx in tqdm(train.index):\n    text = train.loc[idx, 'text'].lower().replace('!?.,;:-\"\\'$%^&*#@{}[]|\\//(/)\"', ' ').strip().replace('  ', ' ')\n    index = text.find(train.loc[idx, 'cleaned_label'])\n    if index >= 0:\n        TRAIN_DATA.append(\n            (text,\n                {\"entities\": \n                    [\n                        (index, index+len(text), \"DATASET\")\n                    ]\n                }\n            )\n            \n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(TRAIN_DATA)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Start training"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nspacy.require_gpu()\n\nprdnlp = train_spacy(TRAIN_DATA, 3)\n\nmodelfile = 'ner'\nprdnlp.to_disk(modelfile)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test the resulting model"},{"metadata":{"trusted":true},"cell_type":"code","source":"for text, annotations in tqdm(TRAIN_DATA[:100]):\n\n    doc = prdnlp(text)\n    for ent in doc.ents:\n        print(ent.text, ent.start_char, ent.end_char, ent.label_)\n        \n        print('-----------------------------------')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Okay, we can see that model trained on 300 texts cannot find the dataset names from the texts. Have you any idea for improvement?"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}