{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-info\">\n\n<b>Note:</b> This is a work in progress notebook!   \n\n</div>\n\n*Also, check my \"Fast solution. Coleridge Initiative: Baseline\" which i used to generate the first version of the wordmodel dataset*\n\n*(https://www.kaggle.com/nikitakudriashov/fast-solution-coleridge-initiative-baseline)*","metadata":{}},{"cell_type":"markdown","source":"# Coleridge Initiative solution\n\n### Named Entity Recognition task (BIO labeling + Glove Embadding + biLSTM + CDF layer)\n----\n\n\n#### 👋 Hi, I am new to NLP and espessially to Named Entity Recognition.\n \n#### 🎓 Actually, I took this task just to get more envolved into this sphere and to sort out my knowledge.\n \n#### 💸 I hope that it will be useful for you.\n \n#### 🧠 I will be glad to hear your advices and best practices examples\n \n#### 👍 And ofcourse I'll be happy to geet your upvotes, enjoy","metadata":{}},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nfrom functools import partial\nimport pandas as pd\nimport numpy as np\nimport gc\nimport re\nimport json\nimport nltk\n\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom sklearn.model_selection import train_test_split\n\npd.set_option('display.max_rows', 1000)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('display.max_colwidth', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-07T13:35:28.996611Z","iopub.execute_input":"2021-06-07T13:35:28.997238Z","iopub.status.idle":"2021-06-07T13:35:30.469912Z","shell.execute_reply.started":"2021-06-07T13:35:28.997127Z","shell.execute_reply":"2021-06-07T13:35:30.469198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_raw_txt(_id:str, path=\"train\"):\n    \"\"\"\n    Gets row publication text by its _id\n    :param _id: publication string id \n    :param path: root directory to the publication json files - train/test\n    :return: raw string of text\n    \"\"\"\n    _d = json.loads(open(f\"{PATH}/{path}/{_id}.json\").read())\n    return \" \".join([i[\"text\"] for i in _d])\n\ndef clean_text(txt:str):\n    \"\"\"\n    Code snippet from the main competition page\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef get_lbl_positions(label:str, text:str):\n    \"\"\"\n    Method provides the position of the 1st and the last word of the label in the given text\n    :return: None if no match or _f,_l - for the first and the last word positions\n    \"\"\"\n    _f = re.search(label+\" \",text)\n    if _f:\n        _f = len(word_tokenize(text[:_f.start()]))\n        _l = _f + len(word_tokenize(label))\n        return int(_f), int(_l)\n        \ndef encode_bio_by_position(text:list, positions:list, padding:int):\n    \"\"\"\n    Returns bio encoded string by its features\n    :param positions: list of tuples\n    \"\"\"\n    bio = np.zeros((padding,),dtype=np.int8)\n    for _f, _l in positions:\n        bio[_f] = 2 #B\n        bio[_f+1:_l] = 1 #I\n    return bio","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:35:30.471489Z","iopub.execute_input":"2021-06-07T13:35:30.472029Z","iopub.status.idle":"2021-06-07T13:35:30.483065Z","shell.execute_reply.started":"2021-06-07T13:35:30.471987Z","shell.execute_reply":"2021-06-07T13:35:30.482015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### Train set transformation\nOn the previouse steps i have generated the wordset from the matches with the text in the train_df. Now we need to relable it with the aim of:\n* It is not fully labeled (not all matches are taken into account)\n* I will group it in a form where Id is a primar key, so our dataset length equels to the number of publications\n* Our train dataset will be putted into the same form as the test one wil be (Id & PredictionString)","metadata":{}},{"cell_type":"code","source":"PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\ntqdm_params = dict(bar_format='{desc}{bar} [ {n} / {total} (remaining: {remaining}) ]', colour=\"darkgreen\")\nwordmodel = pd.read_csv(\"../input/wordmodel/wordmodel.csv\").values.squeeze()\nwordmodel = sorted(wordmodel, key=len)\n\ntqdm.pandas(desc=\"Text download from JSON files status :\",**tqdm_params)\ndf_train = pd.DataFrame(np.unique(pd.read_csv(f\"{PATH}train.csv\")[[\"Id\"]]), columns = [\"Id\"])\ndf_train['text'] = df_train['Id'].progress_apply(partial(get_raw_txt))\n\ntqdm.pandas(desc=\"Text cleaning with clean_text function :\",**tqdm_params)\ndf_train['text'] = df_train['text'].progress_apply(clean_text)\n\ndf_train['PredictionString'] = ''\ndesc=\"Grouping & relabeling by the wordmodel :\"\nfor i in tqdm(df_train.index, desc=desc ,**tqdm_params):\n    for ws in wordmodel:\n        if ws in df_train.loc[i][\"text\"]:\n            df_train.loc[i]['PredictionString'] += ws.strip() + '|'\n    df_train.loc[i]['PredictionString'] = df_train.loc[i]['PredictionString'][:-1]\n\ndf_train = df_train.drop(\"text\",axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:35:30.485238Z","iopub.execute_input":"2021-06-07T13:35:30.485503Z","iopub.status.idle":"2021-06-07T13:41:39.634253Z","shell.execute_reply.started":"2021-06-07T13:35:30.485478Z","shell.execute_reply":"2021-06-07T13:41:39.632967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.sample(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:41:39.636515Z","iopub.execute_input":"2021-06-07T13:41:39.636964Z","iopub.status.idle":"2021-06-07T13:41:39.658588Z","shell.execute_reply.started":"2021-06-07T13:41:39.636918Z","shell.execute_reply":"2021-06-07T13:41:39.657491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### BIO Labeling\nI did the BIO labeling with the self-written code, to have more control of it. It looks like not be the optimal solution, may work a bit long. There are some bottlenecks for the labeling:\n* Some sentances have a really huge length after the tokenization. To handle such of them, we simply prune all the values on the positions after the maxlen.\n* When prunning the values, we need to check if our lable exist in non-pruned values or not.\n* We need to collect the BIO-labeled sequences into one sequence, when multipal matches are in text","metadata":{}},{"cell_type":"code","source":"x,y=[],[]\nmaxlen = 100\ndesc=\"X & Y extraction + BIO labeling :\"\ncounter, too_long = 0 , []\nfor _id in tqdm(df_train.index, desc=desc, **tqdm_params):\n    # Getting list of cleaned sentances from text by Id    \n    _txt = [clean_text(_i) for _i in sent_tokenize(get_raw_txt(df_train.loc[_id,\"Id\"]))]\n    # Getting list of labels\n    _labels = df_train.loc[_id,\"PredictionString\"].split('|')\n    for _sent in _txt:\n        _positions = []\n        for _label in _labels:\n            _pos = get_lbl_positions(_label,_sent)\n            # Non-pruned labels check\n            if _pos:\n                if _pos[1] <= maxlen:\n                    _positions.append(_pos)\n        if _positions:\n            x.append(word_tokenize(_sent)[:maxlen])\n            y.append(encode_bio_by_position(_sent,_positions,maxlen))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:41:39.660125Z","iopub.execute_input":"2021-06-07T13:41:39.660448Z","iopub.status.idle":"2021-06-07T13:49:09.553212Z","shell.execute_reply.started":"2021-06-07T13:41:39.660418Z","shell.execute_reply":"2021-06-07T13:49:09.552073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\" \".join(x[0]))\nprint(\" \".join([str(_i) for _i in y[0]]))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:49:09.554797Z","iopub.execute_input":"2021-06-07T13:49:09.555216Z","iopub.status.idle":"2021-06-07T13:49:09.562425Z","shell.execute_reply.started":"2021-06-07T13:49:09.555173Z","shell.execute_reply":"2021-06-07T13:49:09.561411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GloVe embedding\nAgain,i decided to apply Glove embadding manually, without putting it into the Keras layer, to save the model train performance \nI took the Stanford's GloVe 100d word embeddings","metadata":{}},{"cell_type":"code","source":"glove = open('../input/glove6b100dtxt/glove.6B.100d.txt', encoding=\"utf8\")\nglove_dict = {}\n\ndesc=\"Filling up the GloVe dict :\"\nfor _line in tqdm(glove, desc=desc, **tqdm_params):\n    records = _line.split()\n    glove_dict[records[0]] = [float(i) for i in records[1:]]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:49:09.563553Z","iopub.execute_input":"2021-06-07T13:49:09.564072Z","iopub.status.idle":"2021-06-07T13:49:26.816292Z","shell.execute_reply.started":"2021-06-07T13:49:09.564038Z","shell.execute_reply":"2021-06-07T13:49:26.815303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxlen = max([len(i) for i in x])\nwnl = WordNetLemmatizer()\nzero_vector = [0]*100\n\ndesc=\"Embedding with the GloVe vectors :\"\nfor _i in tqdm(range(len(x)), desc=desc, **tqdm_params):\n    for _j in range(len(x[_i])):\n        try:\n            x[_i][_j] = glove_dict[wnl.lemmatize(x[_i][_j])]\n        except KeyError:\n            x[_i][_j] = zero_vector\n\ndesc=\"Padding with the zero vaectors :\"\nfor _i in tqdm(range(len(x)), desc=desc, **tqdm_params):\n    _pad = maxlen - len(x[_i])\n    for _j in range(_pad):\n        x[_i].append(zero_vector)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:49:26.817357Z","iopub.execute_input":"2021-06-07T13:49:26.817612Z","iopub.status.idle":"2021-06-07T13:49:39.47045Z","shell.execute_reply.started":"2021-06-07T13:49:26.817589Z","shell.execute_reply":"2021-06-07T13:49:39.469379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_tr,x_val,y_tr,y_val = train_test_split(np.array(x),np.array(y),test_size=0.3, shuffle=True, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:49:39.472433Z","iopub.execute_input":"2021-06-07T13:49:39.47273Z","iopub.status.idle":"2021-06-07T13:51:15.70921Z","shell.execute_reply.started":"2021-06-07T13:49:39.472701Z","shell.execute_reply":"2021-06-07T13:51:15.708414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x,y,wordmodel, ws, train_test_split, too_long, df_train, zero_vector\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:51:15.710629Z","iopub.execute_input":"2021-06-07T13:51:15.711205Z","iopub.status.idle":"2021-06-07T13:51:16.489484Z","shell.execute_reply.started":"2021-06-07T13:51:15.711162Z","shell.execute_reply":"2021-06-07T13:51:16.488497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%who","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:56:26.01486Z","iopub.execute_input":"2021-06-07T13:56:26.015367Z","iopub.status.idle":"2021-06-07T13:56:26.025799Z","shell.execute_reply.started":"2021-06-07T13:56:26.01533Z","shell.execute_reply":"2021-06-07T13:56:26.024957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del WordNetLemmatizer, clean_text, counter, desc, encode_bio_by_position, get_lbl_positions, get_raw_txt, glove, glove_dict\n#del x,y,wordmodel, ws, train_test_split, too_long, df_train, zero_vector\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:57:21.785719Z","iopub.execute_input":"2021-06-07T13:57:21.786322Z","iopub.status.idle":"2021-06-07T13:57:22.576645Z","shell.execute_reply.started":"2021-06-07T13:57:21.786264Z","shell.execute_reply":"2021-06-07T13:57:22.575378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Model Fitting","metadata":{}},{"cell_type":"code","source":"!pip3 install ../input/keras-crf-whl-files/seqeval-1.2.2-py3-none-any.whl\n!pip3 install ../input/keras-crf-whl-files/keras_crf-0.2.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:57:25.909317Z","iopub.execute_input":"2021-06-07T13:57:25.909704Z","iopub.status.idle":"2021-06-07T13:57:40.530367Z","shell.execute_reply.started":"2021-06-07T13:57:25.909673Z","shell.execute_reply":"2021-06-07T13:57:40.529252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras_crf import CRFModel\nfrom keras import Input, Model\nfrom keras.optimizers import Adam\nfrom keras.layers import Bidirectional,LSTM, Dense,TimeDistributed\n\nfrom keras.metrics import Recall","metadata":{"execution":{"iopub.status.busy":"2021-06-07T13:58:42.74829Z","iopub.execute_input":"2021-06-07T13:58:42.748671Z","iopub.status.idle":"2021-06-07T13:58:42.753582Z","shell.execute_reply.started":"2021-06-07T13:58:42.748641Z","shell.execute_reply":"2021-06-07T13:58:42.752328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = Input(shape=(maxlen,100))\noutputs = Bidirectional(LSTM(units=maxlen, return_sequences=True,recurrent_dropout=0.1))(inputs)\noutputs = TimeDistributed(Dense(maxlen, activation=\"relu\"))(outputs) \nmodel = Model(inputs, outputs)\nmodel = CRFModel(model,3)\n\nmodel.compile(Adam(lr=0.0005,beta_1=0.9,beta_2=0.999,epsilon=1e-07,),metrics=['acc'])\nmodel.fit(x_tr, y_tr, epochs=100, batch_size=2000, validation_data=(x_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:19:28.012429Z","iopub.execute_input":"2021-06-07T14:19:28.013024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"model\")\n!zip -r model.zip model","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:19:53.633503Z","iopub.execute_input":"2021-06-07T12:19:53.634073Z","iopub.status.idle":"2021-06-07T12:20:06.335037Z","shell.execute_reply.started":"2021-06-07T12:19:53.63399Z","shell.execute_reply":"2021-06-07T12:20:06.333415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Prediction on the test set","metadata":{}},{"cell_type":"code","source":"del x_tr,x_val,y_tr,y_val\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:20:25.588578Z","iopub.execute_input":"2021-06-07T12:20:25.589005Z","iopub.status.idle":"2021-06-07T12:20:28.23582Z","shell.execute_reply.started":"2021-06-07T12:20:25.588967Z","shell.execute_reply":"2021-06-07T12:20:28.234454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nimport json\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:20:30.555716Z","iopub.execute_input":"2021-06-07T12:20:30.556138Z","iopub.status.idle":"2021-06-07T12:20:30.562138Z","shell.execute_reply.started":"2021-06-07T12:20:30.556102Z","shell.execute_reply":"2021-06-07T12:20:30.56115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove = open('../input/glove6b100dtxt/glove.6B.100d.txt', encoding=\"utf8\")\nglove_dict = {}\n\ndesc=\"Filling up the GloVe dict :\"\nfor _line in tqdm(glove, desc=desc, **tqdm_params):\n    records = _line.split()\n    glove_dict[records[0]] = [float(i) for i in records[1:]]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:20:31.449015Z","iopub.execute_input":"2021-06-07T12:20:31.449411Z","iopub.status.idle":"2021-06-07T12:20:56.471029Z","shell.execute_reply.started":"2021-06-07T12:20:31.449379Z","shell.execute_reply":"2021-06-07T12:20:56.469483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\ntqdm_params = dict(bar_format='{desc}{bar} [ {n} / {total} (remaining: {remaining}) ]', colour=\"darkgreen\")\nwordmodel = pd.read_csv(\"../input/wordmodel/wordmodel.csv\").values.squeeze()\nwordmodel = sorted(wordmodel, key=len)\n\ntqdm.pandas(desc=\"Text download from JSON files status :\",**tqdm_params)\ndeploy = pd.read_csv(f\"{PATH}sample_submission.csv\")\n\ndeploy.index = deploy['Id']","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:20:56.473205Z","iopub.execute_input":"2021-06-07T12:20:56.47358Z","iopub.status.idle":"2021-06-07T12:20:56.503709Z","shell.execute_reply.started":"2021-06-07T12:20:56.473548Z","shell.execute_reply":"2021-06-07T12:20:56.502506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prediction procedure should be optimised - just the first brief version","metadata":{}},{"cell_type":"code","source":"maxlen = 100\nzero_vector = [0]*100\nwnl = WordNetLemmatizer()\ndebinarize  = lambda arr: {\"100\":\"0\",\"010\":\"1\",\"001\":\"2\"}[\"\".join([str(i) for i in arr])]\n\nfor _id in deploy.index:\n    print(_id)\n    result = []\n    _txt = [clean_text(_i) for _i in sent_tokenize(get_raw_txt(deploy.loc[_id,\"Id\"]))]\n    for _sent in _txt:\n        _sent = word_tokenize(_sent[:maxlen])\n        _row_sent = _sent.copy()\n        \n        #Glove Embadding\n        for _i in range(len(_sent)):\n            try:\n                _sent[_i] = glove_dict[wnl.lemmatize(_sent[_i])]\n            except KeyError:\n                _sent[_i] = zero_vector\n\n        #Padding with the zero vaectors\n        _pad = maxlen - len(_sent)\n        for _j in range(_pad):\n            _sent.append(zero_vector)\n        \n        prediction = model.predict(np.array([_sent])).astype(int)\n        prediction = \"\".join([debinarize(i) for i in prediction[0]])\n        prediction = re.finditer(r\"(21*)\",prediction)\n        if prediction:\n            for i in prediction:\n                # I filter one-words for now, need to be emprooved\n                if i.span()[1]- i.span()[0]>1:\n                    result.append(\" \".join(_row_sent[i.span()[0]:i.span()[1]]))\n    print(\"|\".join(set(result)))\n    print(\"---\")\n    deploy.loc[_id,\"PredictionString\"]=\"|\".join(set(result))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-07T12:20:56.505546Z","iopub.execute_input":"2021-06-07T12:20:56.505877Z","iopub.status.idle":"2021-06-07T12:23:41.553003Z","shell.execute_reply.started":"2021-06-07T12:20:56.505846Z","shell.execute_reply":"2021-06-07T12:23:41.551277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deploy.to_csv(\"deploy.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:43:12.307164Z","iopub.execute_input":"2021-06-05T10:43:12.307581Z","iopub.status.idle":"2021-06-05T10:43:12.317309Z","shell.execute_reply.started":"2021-06-05T10:43:12.307548Z","shell.execute_reply":"2021-06-05T10:43:12.315809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}