{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport time\nimport pickle\nimport json\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\n\n\nimport matplotlib.pyplot as plt\n\n\nfrom collections import defaultdict, Counter\nfrom sklearn.metrics import confusion_matrix, f1_score\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom nltk.stem import WordNetLemmatizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T05:02:53.071537Z","iopub.execute_input":"2021-06-20T05:02:53.072096Z","iopub.status.idle":"2021-06-20T05:02:55.937609Z","shell.execute_reply.started":"2021-06-20T05:02:53.071985Z","shell.execute_reply":"2021-06-20T05:02:55.936777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"config={\n    'label_batch_size': 64,\n    'unlabel_batch_size': 64,\n    'max_seq_len': 100,\n    'glove_dim': 100,\n    'num_labels': 3,\n    'ner_format': \"BIO\",\n    'grad_clip_val': 1,\n    'max_lr': 0.00013,\n    'T': 0.6,\n    'weight_decay': 1e-5,\n    'num_iterations': 1500,\n    'print_every': 50,\n    'eval_every':100,\n    'save_every': 100,\n    'glove_path': '../input/glove6b/glove.6B.100d.txt',\n    'unlabeled_datapath': '../input/dataset/train_data.pkl',\n    'train_folder': \"../input/coleridgeinitiative-show-us-the-data/train\",\n    'test_folder': '../input/coleridgeinitiative-show-us-the-data/test',\n    'ss_folder': '../input/coleridge-semisuperviseddata'\n}\n\nmodel_params={\n    \"pre_embedd_dim\": 100,\n    'word_shape_size': 7,\n    \"word_shape_embedd_dim\": 20,\n    \"hdim\": 128,\n    \"proj_dim\": 512,\n    \"out_dim\": 3,\n    'max_seq_len': 100,\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:02:55.939526Z","iopub.execute_input":"2021-06-20T05:02:55.940123Z","iopub.status.idle":"2021-06-20T05:02:55.948848Z","shell.execute_reply.started":"2021-06-20T05:02:55.940076Z","shell.execute_reply":"2021-06-20T05:02:55.947635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"def update_annotation(annots, s, l):\n    for i in range(s, s+l):\n        if annots[i] == 'I':\n            continue\n        if i == s:\n            annots[i]='B'\n        else:\n            annots[i]='I'\n\ndef get_annotated_data(data):\n    sentence=data['sentence']\n    labels=data['labels'] if data.get('labels', None) else []\n    words=word_tokenize(sentence)\n    annots=['O']*len(words)\n    labels=[word_tokenize(label) for label in labels]\n    \n    for i, word in enumerate(words):\n        for label in labels:\n            if words[i:i+len(label)] == label:\n                update_annotation(annots, i, len(label))\n    return (words, annots)\n\ndef read_file(filepath):\n    with open(filepath) as file:\n        data=file.read()\n    return data\n\ndef read_json_file(filepath):\n    data=json.loads(read_file(filepath))\n    return data\ndef read_pickle(filepath):\n    with open(filepath, 'rb') as file:\n        data=pickle.load(file)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:02:55.950895Z","iopub.execute_input":"2021-06-20T05:02:55.951216Z","iopub.status.idle":"2021-06-20T05:02:55.962139Z","shell.execute_reply.started":"2021-06-20T05:02:55.951186Z","shell.execute_reply":"2021-06-20T05:02:55.960941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PreprocessData:\n    def __init__(self, config):\n        self.train_folder=config['train_folder']\n        self.ss_folder=config['ss_folder']\n        self.glove_path=config['glove_path']\n        self.unlabled_path=config['unlabeled_datapath']\n    \n    def download_glove(self):\n        glove_embeddings={}\n        with open(self.glove_path) as file:\n            for line in file:\n                line=line.split()\n                word=line[0]\n                v=np.array(line[1:]).astype(np.float)\n                glove_embeddings[word]=v\n        return glove_embeddings\n\n    def get_labeled_data(self):\n        pos_data=read_json_file(os.path.join(config['ss_folder'], 'annotation.txt'))\n        neg_data=read_json_file(os.path.join(config['ss_folder'], 'negative_sentences_corrected.txt'))\n        return (pos_data, neg_data)\n    \n    def get_data(self):\n        (pos_data, neg_data)=self.get_labeled_data()\n        unlabeled_data=read_pickle(self.unlabled_path)\n        for data in pos_data:\n            pub_id=data.get('pub_id', None)\n            unlabeled_data.pop(pub_id) if unlabeled_data.get(pub_id, None) else ''\n        for data in neg_data:\n            pub_id=data.get('pub_id', None)\n            unlabeled_data.pop(pub_id) if unlabeled_data.get(pub_id, None) else ''\n        \n        unlabeled_data=list(unlabeled_data.values())\n        pos_data=[get_annotated_data(data) for data in pos_data]\n        neg_data=[get_annotated_data(data) for data in neg_data]\n        \n        glove_embeddings=self.download_glove()\n        labeled_data={\n            'pos_data': pos_data,\n            'neg_data': neg_data\n        }\n        return (glove_embeddings, labeled_data, unlabeled_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:02:55.9638Z","iopub.execute_input":"2021-06-20T05:02:55.964107Z","iopub.status.idle":"2021-06-20T05:02:55.978783Z","shell.execute_reply.started":"2021-06-20T05:02:55.964081Z","shell.execute_reply":"2021-06-20T05:02:55.977611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:39:29.402051Z","iopub.execute_input":"2021-06-17T10:39:29.40263Z","iopub.status.idle":"2021-06-17T10:39:29.419725Z","shell.execute_reply.started":"2021-06-17T10:39:29.402544Z","shell.execute_reply":"2021-06-17T10:39:29.418858Z"}}},{"cell_type":"code","source":"%%time\n(glove_embeddings, labeled_data, unlabeled_data)=PreprocessData(config).get_data()\nlemmatizer=WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:02:55.980428Z","iopub.execute_input":"2021-06-20T05:02:55.980764Z","iopub.status.idle":"2021-06-20T05:03:49.753475Z","shell.execute_reply.started":"2021-06-20T05:02:55.980733Z","shell.execute_reply":"2021-06-20T05:03:49.752133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data iterators","metadata":{}},{"cell_type":"code","source":"def get_word_shape(word):\n    if len(word)==0:\n        return 0\n    ch=word[0]\n    if ch.isupper():\n        return 1\n    elif ch.islower():\n        return 2\n    elif ch.isnumeric():\n        return 3\n    elif ch == ',':\n        return 4\n    elif ch == '(':\n        return 5\n    elif ch==')':\n        return 6\n    return 0","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:03:49.755049Z","iopub.execute_input":"2021-06-20T05:03:49.755418Z","iopub.status.idle":"2021-06-20T05:03:49.764873Z","shell.execute_reply.started":"2021-06-20T05:03:49.755386Z","shell.execute_reply":"2021-06-20T05:03:49.763804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LabeledIterator:\n    def __init__(self,config,glove_embeddings, labeled_data):\n        self.batch_size=config['label_batch_size']\n        self.glove_dim=config['glove_dim']\n        self.num_labels=config['num_labels']\n        self.max_seq_len=config['max_seq_len']\n        self.glove_embeddings=glove_embeddings\n        self.labeled_data=labeled_data\n        \n        \n        pos_data=labeled_data['pos_data']\n        neg_data=labeled_data['neg_data']\n        \n        pos_ids=list(range(len(pos_data)))\n        neg_ids=list(range(len(neg_data)))\n        \n        pos_train_ids=pos_ids[:-1]\n        neg_train_ids=neg_ids[:-1]\n        \n        pos_val_ids=pos_ids[-1:]\n        neg_val_ids=neg_ids[-1:]\n        \n        self.train_data=([data for i, data in enumerate(pos_data) if i in pos_train_ids]+\n                         [data for i, data in enumerate(neg_data) if i in neg_train_ids])\n        \n        self.val_data  =([data for i, data in enumerate(pos_data) if i in pos_val_ids]+\n                         [data for i, data in enumerate(neg_data) if i in neg_val_ids])\n        \n        \n        self.num_train_records=len(self.train_data)\n        self.num_val_records=len(self.val_data)\n    \n    def make_shuffle_data(self, data, shuffle):\n        if shuffle:\n            np.random.shuffle(data)\n            \n    def get_data_by_mode(self, mode, shuffle):\n        if mode=='val':\n            self.make_shuffle_data(self.val_data, shuffle)\n            num_records=self.num_val_records\n            data=self.val_data\n        elif mode == 'train':\n            self.make_shuffle_data(self.train_data, shuffle)\n            num_records=self.num_train_records\n            data=self.train_data\n        return num_records, data\n   \n    def convert_annotation_to_label(self, annot):\n        if annot == 'B':\n            return 1\n        elif annot =='I':\n            return 2\n        return 0\n    \n    def convert_rawdata_to_tensors(self, mbs):\n        mb_size=len(mbs)\n        X=torch.zeros((mb_size, self.max_seq_len, self.glove_dim), dtype=torch.float32)\n        X_embedd=torch.zeros( (mb_size, self.max_seq_len), dtype=torch.long)\n        y_bios=torch.full((mb_size, self.max_seq_len), 3, dtype=torch.long)\n        y_ents=torch.zeros((mb_size, self.max_seq_len), dtype=torch.float32)\n        \n        for i in range(mb_size):\n            (words, annots)=mbs[i]\n            words_len=min(len(words), self.max_seq_len)\n            for j in range(words_len):\n                wshape=get_word_shape(words[j])\n                word=words[j].lower()\n                \n                X_embedd[i][j]=wshape\n                y_bios[i][j]=self.convert_annotation_to_label(annots[j])\n                if y_bios[i][j] == 1 or y_bios[i][j]==2:\n                    y_ents[i][j]=1\n                if word in self.glove_embeddings:\n                    X[i][j]=torch.tensor(self.glove_embeddings[word], dtype=torch.float32)\n                    \n        return (X, X_embedd, y_ents, y_bios)\n    def get_raw_minibatch(self, mode='val', shuffle=False):\n        num_records, data=self.get_data_by_mode(mode, shuffle)\n        for i in range(0, num_records, self.batch_size):\n            yield data[i:i+self.batch_size]\n            \n    def get_minibatch(self, mode='val', shuffle=False):\n        for mbs in self.get_raw_minibatch(mode, shuffle):\n            mbs=self.convert_rawdata_to_tensors(mbs)\n            yield mbs\n    def get_infinite_minibatch(self, mode='val', shuffle=False):\n        while True:\n            for mbs in self.get_minibatch(mode, shuffle):\n                yield mbs\n    def __iter__(self):\n        while True:\n            for mbs in self.get_raw_minibatch('train', True):\n                mbs=self.convert_rawdata_to_tensors(mbs)\n                yield mbs","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:03:49.766347Z","iopub.execute_input":"2021-06-20T05:03:49.766656Z","iopub.status.idle":"2021-06-20T05:03:49.792339Z","shell.execute_reply.started":"2021-06-20T05:03:49.766627Z","shell.execute_reply":"2021-06-20T05:03:49.791078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UnLabeledIterator:\n    def __init__(self, config, glove_embeddings, unlabeled_data):\n        self.batch_size=config['unlabel_batch_size']\n        self.max_seq_len=config['max_seq_len']\n        self.glove_dim=config['glove_dim']\n        self.glove_embeddings=glove_embeddings\n        self.unlabeled_data=unlabeled_data\n        self.num_recods=len(unlabeled_data)\n    def get_raw_minibatch(self):\n        np.random.shuffle(self.unlabeled_data)\n        for i in range(0, self.num_recods, self.batch_size):\n            yield self.unlabeled_data[i:i+self.batch_size]\n    def get_sampled_sentences(self, mbs):\n        sentences=[]\n        tags=[]\n        for i in range( len(mbs) ):\n            pos_sents=mbs[i]['pos_sents']\n            neg_sents=mbs[i]['neg_sents']\n            \n            if len(pos_sents)>0:\n                np.random.shuffle(pos_sents)\n                tags.append(pos_sents[0]['tags'])\n                sentences.append(pos_sents[0]['sentence_words'])\n            if len(neg_sents)>0:\n                np.random.shuffle(neg_sents)\n                tags.append(neg_sents[0]['tags'])\n                sentences.append(neg_sents[0]['sentence_words'])\n        return (sentences, tags)\n        \n    def convert_rawdata_to_tensors(self, mbs):\n        (sentences, tags)=self.get_sampled_sentences(mbs)\n        mbs_size=len(sentences)\n        X=torch.zeros((mbs_size, self.max_seq_len, self.glove_dim), dtype=torch.float32)\n        X_embedd=torch.zeros( (mbs_size, self.max_seq_len), dtype=torch.long)\n        y_ents=torch.zeros((mbs_size, self.max_seq_len), dtype=torch.float32)\n        y_bios=torch.full((mbs_size, self.max_seq_len), 3, dtype=torch.long)\n        slens=torch.zeros(mbs_size, dtype=torch.long)\n        \n        for i, sentence_words in enumerate(sentences):\n            for j in range( min(len(tags[i]), self.max_seq_len) ):\n                wshape=get_word_shape(sentence_words[j])\n                word=sentence_words[j].lower()\n                \n                X_embedd[i][j]=wshape\n                if word in self.glove_embeddings:\n                    X[i][j]=torch.tensor(self.glove_embeddings[word], dtype=torch.float32)\n                    \n                if tags[i][j] == 1 or tags[i][j]==2:\n                    y_ents[i][j]=1.0;\n                y_bios[i][j]=tags[i][j]\n                \n            if len(sentence_words) >= self.max_seq_len:\n                slens[i]=self.max_seq_len\n            else:\n                slens[i]=len(sentence_words)\n        return (X, X_embedd, y_ents, y_bios, slens)\n        \n    def get_minibatch(self):\n        for mbs in self.get_raw_minibatch():\n            mbs=self.convert_rawdata_to_tensors(mbs)\n            yield mbs\n            \n    def get_infinite_minibatch(self):\n        pass\n    \n    def __iter__(self):\n        while True:\n            for mbs in self.get_raw_minibatch():\n                mbs=self.convert_rawdata_to_tensors(mbs)\n                yield mbs","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:03:49.795052Z","iopub.execute_input":"2021-06-20T05:03:49.795509Z","iopub.status.idle":"2021-06-20T05:03:49.819451Z","shell.execute_reply.started":"2021-06-20T05:03:49.795478Z","shell.execute_reply":"2021-06-20T05:03:49.818589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"execution":{"iopub.status.busy":"2021-06-08T05:31:24.526069Z","iopub.execute_input":"2021-06-08T05:31:24.526525Z","iopub.status.idle":"2021-06-08T05:31:24.53855Z","shell.execute_reply.started":"2021-06-08T05:31:24.526502Z","shell.execute_reply":"2021-06-08T05:31:24.537732Z"}}},{"cell_type":"code","source":"class NERLMHead1(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x\n\nclass NERLMHead2(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x\n    \nclass ERLMHead1(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x\n    \nclass ERLMHead2(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:03:49.820987Z","iopub.execute_input":"2021-06-20T05:03:49.82141Z","iopub.status.idle":"2021-06-20T05:03:49.841641Z","shell.execute_reply.started":"2021-06-20T05:03:49.821379Z","shell.execute_reply":"2021-06-20T05:03:49.84029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NERHead(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], params['out_dim'])\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x\n    \nclass ERHead(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.linear=nn.Linear(2*params['hdim'], params['proj_dim'])\n        self.bn=nn.BatchNorm1d(params['proj_dim'])\n        self.dropout=nn.Dropout(0.3)\n        self.relu=nn.ReLU()\n        self.out_layer=nn.Linear(params['proj_dim'], 1)\n    def forward(self, x):\n        x=self.linear(x)\n        x=x.permute(0, 2, 1)\n        x=self.bn(x)\n        x=self.dropout(x)\n        x=self.relu(x)\n        x=x.permute(0, 2, 1)\n        x=self.out_layer(x)\n        return x    \n\nclass Model(nn.Module):\n    def __init__(self, params):\n        super().__init__()\n        self.params=params\n        self.embedd_layer=nn.Embedding(params['word_shape_size'], params['word_shape_embedd_dim'],\n                                       max_norm=1, padding_idx=0)\n        self.gru=nn.GRU(params['pre_embedd_dim'] + params['word_shape_embedd_dim'],\n                        params['hdim'], num_layers=2,\n                        bidirectional=True, dropout=0.3, batch_first=True)\n        \n        self.ner_head=NERHead(params)\n        self.er_head=ERHead(params)\n        \n        self.ner_lm_head1 = NERLMHead1(params)\n        self.ner_lm_head2 = NERLMHead2(params)\n        \n        self.er_lm_head1=ERLMHead1(params)\n        self.er_lm_head2=ERLMHead2(params)\n        \n    def forward(self, x, x_embedd):\n        batch_size=x.shape[0]\n        seq_len=x.shape[1]\n        x_embedd=self.embedd_layer(x_embedd)\n        (h_n, _)=self.gru(torch.cat([x, x_embedd], dim=-1))\n        h_n=h_n.view(batch_size, seq_len, 2, self.params['hdim'] )\n        h1=h_n[:, :, 0, :]\n        h2=h_n[:, :, 1, :]\n        \n        h=torch.cat([h1, h2], dim=-1)\n        y_ent=self.er_head(h)\n        y_bios=self.ner_head(h)\n        \n        y_lm_ent1=self.er_lm_head1(h)\n        y_lm_ent2=self.er_lm_head2(h)\n        \n        y_lm_bios1= self.ner_lm_head1(h)\n        y_lm_bios2= self.ner_lm_head2(h)\n        \n        return {\n            'y_ent': y_ent,\n            'y_bios': y_bios,\n            'y_lm_ent1': y_lm_ent1,\n            'y_lm_ent2': y_lm_ent2,\n            'y_lm_bios1': y_lm_bios1,\n            'y_lm_bios2': y_lm_bios2\n        }","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-20T05:03:49.843094Z","iopub.execute_input":"2021-06-20T05:03:49.84339Z","iopub.status.idle":"2021-06-20T05:03:49.866507Z","shell.execute_reply.started":"2021-06-20T05:03:49.843362Z","shell.execute_reply":"2021-06-20T05:03:49.865059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretraining","metadata":{}},{"cell_type":"code","source":"def evaluate(model):\n    model.eval()\n    true_bio_labels=[];pred_bio_labels=[];\n    labeled_iterator=LabeledIterator(config, glove_embeddings, labeled_data)\n    \n    for mbs in labeled_iterator.get_minibatch(mode='train', shuffle=False):\n        (X, X_embedd, y_ents, y_bios)=mbs\n        with torch.no_grad():\n            y_out=model(X, X_embedd)\n            yhat=y_out['y_bios']\n            yhat=yhat.argmax(dim=-1)\n        \n        batch_size=X.shape[0]\n        \n        y_bios=y_bios.view(-1)\n        yhat=yhat.view(-1)\n        \n        yhat=yhat[y_bios!=3]\n        y_bios=y_bios[y_bios!=3]\n        \n        true_bio_labels+=list(y_bios.numpy())\n        pred_bio_labels+=list(yhat.numpy())\n    \n    cm=confusion_matrix(true_bio_labels, pred_bio_labels)\n    micro_fscore_bio=f1_score(true_bio_labels, pred_bio_labels, average='micro')\n    macro_fscore_bio=f1_score(true_bio_labels, pred_bio_labels, average='macro')\n    \n    return (cm, micro_fscore_bio, macro_fscore_bio)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-20T05:03:49.868643Z","iopub.execute_input":"2021-06-20T05:03:49.869247Z","iopub.status.idle":"2021-06-20T05:03:49.884962Z","shell.execute_reply.started":"2021-06-20T05:03:49.869199Z","shell.execute_reply":"2021-06-20T05:03:49.883607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomLoss:\n    def __init__(self):\n        pass\n    \n    def get_entity_loss(self, slens, y_ents, logyhat_ent0, logyhat_ent1 ):\n        batch_size=y_ents.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            ysum=torch.sum(y_ents[i])\n            if ysum==0:\n                neg_loss+=(-1 * 0.95*logyhat_ent0[i][:slens[i]]).sum() + (-1 * 0.05 * logyhat_ent1[i][:slens[i]]).sum()\n                neg_cnt+=slens[i]\n                continue\n            for j in range(slens[i]):\n                if y_ents[i][j]==1:\n                    pos_loss+=((-1 * 0.95*logyhat_ent1[i][j]) + (-1 * 0.05 * logyhat_ent0[i][j])).sum()\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        loss=pos_loss+neg_loss\n        return loss\n    \n    def get_lm1_entity_loss(self, slens, y_ents, logyhat_lm1_ent1, logyhat_lm1_ent0):\n        batch_size=y_ents.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            seq_len=slens[i]\n            ysum=torch.sum(y_ents[i][1: seq_len])\n            if ysum==0:\n                neg_loss+=(-1 * 0.95*logyhat_lm1_ent0[i][:seq_len-1]).sum() + (-1 * 0.05 * logyhat_lm1_ent1[i][:seq_len-1]).sum()\n                neg_cnt+=seq_len\n                continue\n            \n            for j in range(seq_len-1):\n                if y_ents[i][j+1]==1: # If the next label==1; \n                    pos_loss+=((-1 * 0.95*logyhat_lm1_ent1[i][j]) + (-1 * 0.05 * logyhat_lm1_ent0[i][j])).sum()\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        loss=pos_loss+neg_loss\n        return loss\n    \n    def get_lm2_entity_loss(self, slens, y_ents, logyhat_lm2_ent1, logyhat_lm2_ent0):\n        batch_size=y_ents.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            seq_len=slens[i]\n            ysum=torch.sum(y_ents[i][0: seq_len-1])\n            if ysum==0:\n                neg_loss+=(-1 * 0.95*logyhat_lm2_ent0[i][1:seq_len]).sum() + (-1 * 0.05 * logyhat_lm2_ent1[i][1:seq_len]).sum()\n                neg_cnt+=(seq_len-1)\n                continue\n                \n            for j in range(1, seq_len):\n                if y_ents[i][j-1]==1: # If the previous label==1; \n                    pos_loss+=((-1 * 0.95*logyhat_lm2_ent1[i][j]) + (-1 * 0.05 * logyhat_lm2_ent0[i][j])).sum()\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        loss=pos_loss+neg_loss\n        return loss\n    \n    \n    def get_bios_loss(self, slens, y_bios, logyhat_bios):\n        batch_size=y_bios.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n\n        for i in range(batch_size):\n            ysum=torch.sum(y_bios[i][:slens[i]].sum())\n            if ysum==0:\n                neg_loss+=(-1*0.95*logyhat_bios[i][:slens[i]][0].sum()) + \\\n                (-1*0.025*logyhat_bios[i][:slens[i]][1].sum()) + \\\n                (-1*0.025*logyhat_bios[i][:slens[i]][2].sum())\n                neg_cnt+=slens[i].item()\n                continue\n                \n            for j in range(slens[i]):\n                if y_bios[i][j]==1:\n                    pos_loss+=(-1*0.9*logyhat_bios[i][j][1].sum()) + \\\n                    (-1*0.08*logyhat_bios[i][j][2].sum()) + \\\n                    (-1*0.02*logyhat_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    \n                elif y_bios[i][j]==2:\n                    pos_loss+=(-1*0.9*logyhat_bios[i][j][2].sum()) + \\\n                    (-1*0.08*logyhat_bios[i][j][1].sum()) + \\\n                    (-1*0.02*logyhat_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        \n        loss=pos_loss+neg_loss\n        return loss\n    \n    def get_lm1_bios_loss(self, slens, y_bios, logyhat_lm1_bios):\n        batch_size=y_bios.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            seq_len=slens[i].item()\n            ysum=torch.sum(y_bios[i][1:seq_len].sum())\n            if ysum==0:\n                neg_loss+=(-1*0.95*logyhat_lm1_bios[i][:seq_len-1][0].sum()) + \\\n                (-1*0.025*logyhat_lm1_bios[i][:seq_len-1][1].sum()) + \\\n                (-1*0.025*logyhat_lm1_bios[i][:seq_len-1][2].sum())\n                neg_cnt+=(seq_len-1)\n                continue\n                \n            for j in range(seq_len-1):\n                if y_bios[i][j+1]==1: # Next Label is 1\n                    pos_loss+=(-1*0.9*logyhat_lm1_bios[i][j][1].sum()) + \\\n                    (-1*0.08*logyhat_lm1_bios[i][j][2].sum()) + \\\n                    (-1*0.02*logyhat_lm1_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    \n                elif y_bios[i][j+1]==2:# Next Label is 2\n                    pos_loss+=(-1*0.9*logyhat_lm1_bios[i][j][2].sum()) + \\\n                    (-1*0.08*logyhat_lm1_bios[i][j][1].sum()) + \\\n                    (-1*0.02*logyhat_lm1_bios[i][j][0].sum())\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        \n        loss=pos_loss+neg_loss\n        return loss\n\n    \n    def get_lm2_bios_loss(self, slens, y_bios, logyhat_lm2_bios):\n        batch_size=y_bios.shape[0]\n        loss=torch.tensor(0.0)\n        pos_loss=torch.tensor(0.0); neg_loss=torch.tensor(0.0)\n        pos_cnt=0; neg_cnt=0\n        \n        for i in range(batch_size):\n            seq_len=slens[i].item()\n            ysum=torch.sum(y_bios[i][0:seq_len-1].sum())\n            if ysum==0:\n                neg_loss+=(-1*0.95*logyhat_lm2_bios[i][1:seq_len][0].sum()) + \\\n                (-1*0.025*logyhat_lm2_bios[i][1:seq_len][1].sum()) + \\\n                (-1*0.025*logyhat_lm2_bios[i][1:seq_len][2].sum())\n                neg_cnt+=seq_len\n                continue\n                \n            for j in range(1, seq_len):\n                if y_bios[i][j-1]==1: # Next Label is 1\n                    pos_loss+=(-1*0.9*logyhat_lm2_bios[i][j][1].sum()) + \\\n                    (-1*0.08*logyhat_lm2_bios[i][j][2].sum()) + \\\n                    (-1*0.02*logyhat_lm2_bios[i][j][0].sum())\n                    pos_cnt+=1\n                    \n                elif y_bios[i][j-1]==2:# Next Label is 2\n                    pos_loss+=(-1*0.9*logyhat_lm2_bios[i][j][2].sum()) + \\\n                    (-1*0.08*logyhat_lm2_bios[i][j][1].sum()) + \\\n                    (-1*0.02*logyhat_lm2_bios[i][j][0].sum())\n                    pos_cnt+=1\n        pos_loss/=max(1, pos_cnt)\n        neg_loss/=max(1, neg_cnt)\n        \n        loss=pos_loss+neg_loss\n        return loss\n    \n    \n    def get_constraint_loss(self, slens, logyhat_bios, sm_yhat_bios,  logyhat_ent0, logyhat_ent1):\n        loss=torch.tensor(0.0)\n        batch_size=logyhat_bios.shape[0]\n        for i in range(batch_size):\n            cur_loss=torch.tensor(0.0)\n            for j in range(slens[i]):\n                cur_loss+=torch.abs(logyhat_bios[i][j][0] - logyhat_ent0[i][j][0])\n                cur_loss += torch.abs(torch.log(1-sm_yhat_bios[i][j][0]+1e-9) - logyhat_ent1[i][j][0])\n                \n            loss+=(cur_loss/max(1, slens[i]))\n        loss/=max(batch_size, 1)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:03:49.886839Z","iopub.execute_input":"2021-06-20T05:03:49.887338Z","iopub.status.idle":"2021-06-20T05:03:49.94195Z","shell.execute_reply.started":"2021-06-20T05:03:49.887296Z","shell.execute_reply":"2021-06-20T05:03:49.940484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pretraining:\n    def __init__(self, model):\n        self.model=model\n        self.iter_count=0\n        self.total_loss=0.0\n        self.ent_loss=0.0\n        self.bios_loss=0.0\n        self.loss_lm_ent1=0.0\n        self.loss_lm_ent2=0.0\n        self.loss_bios_lm1=0.0\n        self.loss_bios_lm2=0.0\n        \n        self.constraint_loss=0.0\n        self.save_total_loss=0.0\n        \n        self.loss_=[]\n        self.ent_loss_=[]\n        self.bios_loss_=[]\n        self.constraint_loss_=[]\n        self.loss_lm_ent1_=[]\n        self.loss_lm_ent2_=[]\n        self.loss_bios_lm1_=[]\n        self.loss_bios_lm2_=[]\n        \n        self.criterion1=nn.BCEWithLogitsLoss(reduction='mean')\n        self.criterion2=nn.CrossEntropyLoss(ignore_index=3, reduction='mean')\n        \n        self.logsoftmax=nn.LogSoftmax(dim=-1)\n        self.softmax=nn.Softmax(dim=-1)\n        self.logsigmoid=nn.LogSigmoid()\n        self.sigmoid=nn.Sigmoid()\n        \n        self.customLoss=CustomLoss()\n        self.optimizer=torch.optim.AdamW(model.parameters(), lr=config['max_lr'], weight_decay=config['weight_decay'])\n        self.schedular=torch.optim.lr_scheduler.OneCycleLR(self.optimizer,\n                                                           max_lr=config['max_lr'], \n                                                           total_steps=config['num_iterations'])\n        \n        \n        #self.schedular=torch.optim.lr_scheduler.StepLR(self.optimizer, 1, gamma=1.02)\n        self.unlabelIterator=iter(UnLabeledIterator(config, glove_embeddings, unlabeled_data))\n        self.labeledIterator=iter(LabeledIterator(config, glove_embeddings, labeled_data))\n    \n    def train_ops(self, mbs):\n        self.model.train()\n        \n        (X, X_embedd, y_ents, y_bios, slens)=mbs\n        batch_size=X.shape[0]\n        seq_len=X.shape[1]\n        \n        yout=self.model(X, X_embedd)\n        \n        yhat_ent=yout['y_ent']\n        yhat_bios=yout['y_bios']\n        yhat_lm_ent1=yout['y_lm_ent1']\n        yhat_lm_ent2=yout['y_lm_ent2']\n        yhat_lm_bios1=yout['y_lm_bios1']\n        yhat_lm_bios2=yout['y_lm_bios2']\n        \n        \n        \n        logyhat_ent1=torch.log(self.sigmoid(yhat_ent))\n        logyhat_ent0=torch.log(1-self.sigmoid(yhat_ent))\n        \n        logyhat_lm1_ent1=torch.log(self.sigmoid(yhat_lm_ent1)) # LM1\n        logyhat_lm1_ent0=torch.log(1-self.sigmoid(yhat_lm_ent1))# LM1\n        \n        \n        logyhat_lm2_ent1=torch.log(self.sigmoid(yhat_lm_ent2))# LM2\n        logyhat_lm2_ent0=torch.log(1-self.sigmoid(yhat_lm_ent2))# LM2\n        \n        \n        logyhat_bios=self.logsoftmax(yhat_bios)\n        sm_yhat_bios =self.softmax(yhat_bios) #required for the Constraint loss\n        \n        \n        logyhat_lm1_bios=self.logsoftmax(yhat_lm_bios1)#LM1\n        logyhat_lm2_bios=self.logsoftmax(yhat_lm_bios2)#LM2\n        \n        \n        loss_ents=self.customLoss.get_entity_loss(slens, y_ents, logyhat_ent0, logyhat_ent1)\n        loss_ent_lm1=self.customLoss.get_lm1_entity_loss(slens, y_ents, logyhat_lm1_ent1, logyhat_lm1_ent0)\n        loss_ent_lm2=self.customLoss.get_lm2_entity_loss(slens, y_ents, logyhat_lm2_ent1, logyhat_lm2_ent0)\n        loss_bios=self.customLoss.get_bios_loss(slens, y_bios, logyhat_bios)\n        loss_bios_lm1=self.customLoss.get_lm1_bios_loss(slens, y_bios, logyhat_lm1_bios)\n        loss_bios_lm2=self.customLoss.get_lm2_bios_loss(slens, y_bios, logyhat_lm2_bios)\n        loss_constraint=self.customLoss.get_constraint_loss(slens, logyhat_bios, sm_yhat_bios,  logyhat_ent0, logyhat_ent1)\n\n        loss=(loss_ents+loss_bios+loss_constraint) + 0.7*(loss_ent_lm1+loss_ent_lm2+loss_bios_lm1+loss_bios_lm2)\n        loss/=7\n        \n        self.optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(self.model.parameters(), config['grad_clip_val'])\n        self.optimizer.step()\n        self.schedular.step()\n        \n        losses={\n            'loss': loss.item(),\n            'loss_ents': loss_ents.item(),\n            'loss_bios': loss_bios.item(),\n            'loss_constraint': loss_constraint.item(),\n            'loss_ent_lm1': loss_ent_lm1.item(),\n            'loss_ent_lm2': loss_ent_lm2.item(),\n            'loss_bios_lm1': loss_bios_lm1.item(),\n            'loss_bios_lm2': loss_bios_lm2.item()\n        }\n        return losses\n    \n    def train(self):\n        best_loss=None\n        best_fscore=None\n        t1=time.time()\n        self.model.train()\n        while self.iter_count<config['num_iterations']:\n            self.iter_count+=1\n            mbs=next(self.unlabelIterator)\n            losses = self.train_ops(mbs)\n            \n            self.total_loss+=losses['loss']\n            self.ent_loss+=losses['loss_ents']\n            self.bios_loss+=losses['loss_bios']\n            self.constraint_loss+=losses['loss_constraint']\n            \n            self.loss_lm_ent1+=losses['loss_ent_lm1']\n            self.loss_lm_ent2+=losses['loss_ent_lm2']\n            \n            self.loss_bios_lm1+=losses['loss_bios_lm1']\n            self.loss_bios_lm2+=losses['loss_bios_lm2']\n            \n            self.save_total_loss+=losses['loss']\n            \n            \n            self.loss_.append(losses['loss'])\n            self.ent_loss_.append(losses['loss_ents'])\n            self.bios_loss_.append(losses['loss_bios'])\n            self.constraint_loss_.append(losses['loss_constraint'])\n            \n            self.loss_lm_ent1_.append(losses['loss_ent_lm1'])\n            self.loss_lm_ent2_.append(losses['loss_ent_lm2'])\n            self.loss_bios_lm1_.append(losses['loss_bios_lm1'])\n            self.loss_bios_lm2_.append(losses['loss_bios_lm2'])\n            \n            if self.iter_count%config['save_every'] == 0:\n                torch.save(model, 'model_{}.pt'.format(self.iter_count))\n                (cm, micro_fscore_bio, macro_fscore_bio)=evaluate(model)\n                self.save_total_loss/=config['save_every']\n                if (best_loss is None) or (best_loss > self.save_total_loss):\n                    torch.save(model, 'best_loss_model.pt')\n                    best_loss = self.save_total_loss\n                    self.save_total_loss=0.0\n                if (best_fscore is None) or (best_fscore < macro_fscore_bio):\n                    torch.save(model, 'best_fscore_model.pt')\n                    best_fscore=macro_fscore_bio\n                \n                print(\"=====\"*10)\n                print(\"Saving Best Model\")\n                print(\"Best Loss:{:.4f}\".format(best_loss))\n                print(\"Best F-Score:{:.4f}\".format(best_fscore))\n            \n            if self.iter_count%config['eval_every']==0:\n                print(\"Evaluating:\")\n                print(\"======\"*10)\n                (cm, micro_fscore_bio, macro_fscore_bio)=evaluate(model)\n                print(\"Confusion Matrix:\")\n                print(cm)\n                \n                print(\"Micro F-Score ==> {:.4f}\".format(micro_fscore_bio))\n                print(\"Macro F-Score ==> {:.4f}\".format(macro_fscore_bio))\n                print(\"======\"*10)\n                print()\n                torch.save(model, 'model.pt')\n\n            if self.iter_count%config['print_every']==0:\n                t2=time.time()\n                print(\"====\"*10)\n                print(\"Iteration:{} | Time Taken:{:.1f}\".format(self.iter_count, (t2-t1)/60))\n                print()\n                print(\"Total Loss:{:.4f}\".format(self.total_loss/config['print_every']))\n                print(\"Entity Loss:{:.4f}\".format(self.ent_loss/config['print_every']))\n                print(\"Bios Loss:{:.4f}\".format(self.bios_loss/config['print_every']))\n                print(\"Constraint Loss:{:.4f}\".format(self.constraint_loss/config['print_every']))\n                print()\n                print(\"LM-1 Entity Loss:{:.4f}\".format(self.loss_lm_ent1/config['print_every']))\n                print(\"LM-2 Entity Loss:{:.4f}\".format(self.loss_lm_ent2/config['print_every']))\n                \n                print(\"LM-1 BIOS Loss:{:.4f}\".format(self.loss_bios_lm1/config['print_every']))\n                print(\"LM-2 BIOS Loss:{:.4f}\".format(self.loss_bios_lm2/config['print_every']))\n                \n                \n                t1=time.time()\n                self.total_loss=0\n                self.ent_loss=0\n                self.bios_loss=0\n                self.constraint_loss=0\n                \n                self.loss_lm_ent1=0\n                self.loss_lm_ent2=0\n                \n                self.loss_bios_lm1=0\n                self.loss_bios_lm2=0\n                \n                print()\n    def lr_range_test(self):\n        min_lr=1e-5\n        max_lr=2e-3\n        cnt=0\n        lrs=[]\n        losses=[]\n        while min_lr < max_lr:\n            cnt+=1\n            min_lr=self.schedular.get_last_lr()[0]\n            mbs=next(self.unlabelIterator)\n            yout=self.train_ops(mbs)\n            losses.append(yout['loss'])\n            lrs.append(min_lr)\n            if cnt%10==0:\n                print(\"Iteration:{} | LR:{} | Loss:{:3f}\".format(cnt, min_lr, yout['loss']))\n        return (lrs, losses)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-20T05:03:49.943418Z","iopub.execute_input":"2021-06-20T05:03:49.943865Z","iopub.status.idle":"2021-06-20T05:03:49.986621Z","shell.execute_reply.started":"2021-06-20T05:03:49.943832Z","shell.execute_reply":"2021-06-20T05:03:49.985347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Model(model_params)\npreTrainer=Pretraining(model)\n\npreTrainer.train()\ntorch.save(model, 'model.pt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax=plt.subplots(2, 2, figsize=(15, 7), sharex=True, sharey=True)\nax[0][0].plot(range(len(preTrainer.loss_)) , preTrainer.loss_)\nax[0][1].plot(range(len(preTrainer.ent_loss_)) , preTrainer.ent_loss_)\nax[1][0].plot(range(len(preTrainer.bios_loss_)) , preTrainer.bios_loss_)\nax[1][1].plot(range(len(preTrainer.constraint_loss_)) , preTrainer.constraint_loss_)\n\nax[0][0].set_title(\"Total Loss\")\nax[0][1].set_title(\"Entity Loss\")\nax[1][0].set_title(\"Bios Loss\")\nax[1][1].set_title(\"Constraint Loss\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax=plt.subplots(2, 2, figsize=(15, 7), sharex=True, sharey=True)\nax[0][0].plot(range(len(preTrainer.loss_lm_ent1_)) , preTrainer.loss_lm_ent1_)\nax[0][1].plot(range(len(preTrainer.loss_lm_ent2_)) , preTrainer.loss_lm_ent2_)\nax[1][0].plot(range(len(preTrainer.loss_bios_lm1_)) , preTrainer.loss_bios_lm1_)\nax[1][1].plot(range(len(preTrainer.loss_bios_lm2_)) , preTrainer.loss_bios_lm2_)\n\nax[0][0].set_title(\"Entity Loss - LM1\")\nax[0][1].set_title(\"Entity Loss - LM2\")\nax[1][0].set_title(\"Bios Loss - LM1\")\nax[1][1].set_title(\"Bios Loss - LM2\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}