{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from transformers import AutoTokenizer\n\nbert_base_path = '../input/huggingface-bert/bert-base-uncased'\nroberta_base_path = '../input/huggingface-roberta/roberta-base'\n\nbert_tokenizer = AutoTokenizer.from_pretrained(bert_base_path , do_lower_case=True)\nroberta_tokenizer = AutoTokenizer.from_pretrained(roberta_base_path , do_lower_case=True)\n\n##########\n\nexample_sentence = 'Differences in Outcomes for Female and Male Students in Special Education'\nexample_label = 'national education longitudinal study'\n\n##########\n\nprint(\"Bert tokenized sentence:\",bert_tokenizer.tokenize(example_sentence))\nprint(\"Bert tokenized label:\",bert_tokenizer.tokenize(example_label))\n\nprint(\"\")\nprint(\"Roberta  tokenized sentence:\", roberta_tokenizer.tokenize(example_sentence))\nprint(\"Roberta tokenized sentence:\",roberta_tokenizer.tokenize(example_label))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:07.100948Z","iopub.execute_input":"2021-06-24T16:14:07.101270Z","iopub.status.idle":"2021-06-24T16:14:07.824822Z","shell.execute_reply.started":"2021-06-24T16:14:07.101240Z","shell.execute_reply":"2021-06-24T16:14:07.824060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"\nfrom nltk.tokenize import sent_tokenize\nfrom torch.utils.data import DataLoader\nfrom transformers import AutoTokenizer \nfrom sklearn.model_selection import train_test_split   \nfrom tqdm import tqdm\nimport torch \nimport os\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nimport  transformers, glob, datetime, warnings, re, json, random, warnings\nfrom transformers import RobertaForTokenClassification\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:09.740723Z","iopub.execute_input":"2021-06-24T16:14:09.741056Z","iopub.status.idle":"2021-06-24T16:14:10.252443Z","shell.execute_reply.started":"2021-06-24T16:14:09.741023Z","shell.execute_reply":"2021-06-24T16:14:10.251607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_path = '../input/huggingface-roberta/roberta-base'\ntrain_path = '/kaggle/input/coleridgeinitiative-show-us-the-data/train/'\ntest_path = '/kaggle/input/coleridgeinitiative-show-us-the-data/test/*'\nmodel_path = '../input/coleridgemodels/'+ 'model_roberta_base.bin'\n    \nconfig = {'MAX_LEN':128,\n          'tokenizer': AutoTokenizer.from_pretrained(roberta_path , do_lower_case=True),\n          'batch_size':5,\n          'Epoch': 2,\n          'train_path':train_path,\n          'test_path':test_path, \n          'device': 'cuda' if torch.cuda.is_available() else 'cpu',\n          'model_path':model_path,\n          'model_name':'model_roberta_base.bin'\n         }","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:12.589016Z","iopub.execute_input":"2021-06-24T16:14:12.589355Z","iopub.status.idle":"2021-06-24T16:14:12.739782Z","shell.execute_reply.started":"2021-06-24T16:14:12.589324Z","shell.execute_reply":"2021-06-24T16:14:12.738956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading training data","metadata":{}},{"cell_type":"code","source":"#import data\ntrain=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:15.758609Z","iopub.execute_input":"2021-06-24T16:14:15.758939Z","iopub.status.idle":"2021-06-24T16:14:15.889166Z","shell.execute_reply.started":"2021-06-24T16:14:15.758901Z","shell.execute_reply":"2021-06-24T16:14:15.888351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Labels combination\ntrain_df=train.groupby(['Id']).agg(label_count=('cleaned_label', 'count') , label = ('cleaned_label', '|'.join)).reset_index()\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:17.885384Z","iopub.execute_input":"2021-06-24T16:14:17.885713Z","iopub.status.idle":"2021-06-24T16:14:18.038908Z","shell.execute_reply.started":"2021-06-24T16:14:17.885679Z","shell.execute_reply":"2021-06-24T16:14:18.037964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:21.308982Z","iopub.execute_input":"2021-06-24T16:14:21.309338Z","iopub.status.idle":"2021-06-24T16:14:21.315057Z","shell.execute_reply.started":"2021-06-24T16:14:21.309306Z","shell.execute_reply":"2021-06-24T16:14:21.314001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import json data and prepare","metadata":{}},{"cell_type":"code","source":"def import_files(df, path):\n    '''This function reads all json train/test files: theoutput is dictionary with Id as key and the remaining content as values'''\n    text = {}\n    for i, rec_id in tqdm(enumerate(df.Id), total = len(df.Id)):\n        location = f'{path}{rec_id}.json'\n\n        with open(location, 'r') as f:\n            text[rec_id] = json.load(f)\n        \n    print(\"All files read\")\n    end = datetime.datetime.now()\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:22.581783Z","iopub.execute_input":"2021-06-24T16:14:22.582144Z","iopub.status.idle":"2021-06-24T16:14:22.587947Z","shell.execute_reply.started":"2021-06-24T16:14:22.582113Z","shell.execute_reply":"2021-06-24T16:14:22.586951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time \ntrain_data=import_files(df=train_df, path=train_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:14:24.908673Z","iopub.execute_input":"2021-06-24T16:14:24.909016Z","iopub.status.idle":"2021-06-24T16:15:13.998535Z","shell.execute_reply.started":"2021-06-24T16:14:24.908982Z","shell.execute_reply":"2021-06-24T16:15:13.997742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(train_data.keys())\n#print(train_data.values())\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T08:03:54.695966Z","iopub.execute_input":"2021-06-22T08:03:54.696413Z","iopub.status.idle":"2021-06-22T08:03:58.894842Z","shell.execute_reply.started":"2021-06-22T08:03:54.696383Z","shell.execute_reply":"2021-06-22T08:03:58.89297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\ndef data_joining(data_dict_id):\n    '''\n    This function is to join all the text data from different sections in the json to a single text file. \n    '''\n    data_length = len(data_dict_id)\n\n    #     temp = [clean_text(data_dict_id[i]['text']) for i in range(data_length)]\n    temp = [data_dict_id[i]['text'] for i in range(data_length)]\n    temp = '. '.join(temp)\n    \n    return temp\ndef make_shorter_sentence(sentence):\n    sent_tokenized = sent_tokenize(sentence)\n    \n    max_length = config['MAX_LEN']\n    overlap = 20\n    \n    final_sentences = []\n    \n    for tokenized_sent in sent_tokenized:\n        sent_tokenized_clean = clean_text(tokenized_sent)\n        sent_tokenized_clean = sent_tokenized_clean.replace('.','').rstrip() \n        \n        tok_sent = sent_tokenized_clean.split(\" \")\n        \n        if len(tok_sent)<max_length:\n            final_sentences.append(sent_tokenized_clean)\n        else :\n\n            start = 0\n            end = len(tok_sent)\n            \n            for i in range(start, end, max_length-overlap):\n                temp = tok_sent[i: (i + max_length)]\n                final_sentences.append(\" \".join(i for i in temp))\n\n    return final_sentences\ndef form_labels(sentence, labels_list):\n    '''\n    This function labels the training data \n    '''\n    matched_kwords = []\n    matched_token = []\n    un_matched_kwords = []\n    label = []\n\n    # Since there are many sentences which are more than 512. Let's make the max length of all\n    # the sentences be 64\n    tokens = make_shorter_sentence(sentence)\n    \n    for tok in tokens:    \n        tok_split = config['tokenizer'].tokenize(\" \" + tok)\n        \n        z = np.array(['O'] * len(tok_split)) # Create final label == len(tokens) of each sentence\n        matched_keywords = 0 # Initially no kword matched    \n\n        for kword in labels_list:\n            kword_split = config['tokenizer'].tokenize(\" \" + kword)\n            for i in range(len(tok_split)):\n                if tok_split[i: (i + len(kword_split))] == kword_split:\n                    matched_keywords += 1\n\n                    if (len(kword_split) == 1):\n                        z[i] = 'B'\n                    else:\n                        z[i] = 'B'\n                        z[(i+1) : (i+ len(kword_split))]= 'B'\n\n                    if matched_keywords >1:\n                        label[-1] = (z.tolist())\n                        matched_token[-1] = tok\n                        matched_kwords[-1].append(kword)\n                    else:\n                        label.append(z.tolist())\n                        matched_token.append(tok)\n                        matched_kwords.append([kword])\n                    #print(label[-1])\n                    #print(\"\")\n    #                 break\n                else:\n                    un_matched_kwords.append(tok)\n                \n    return matched_token, matched_kwords, label, un_matched_kwords\n\ndef labelling(dataset, data_dict):\n    \n    Id_list_ = []\n    sentences_ = []\n    key_ = []\n    labels_ = []\n    un_mat = []\n    un_matched_reviews = 0\n\n    for i, Id in tqdm(enumerate(dataset.Id), total=len(dataset.Id)):\n\n        \n        sentence = data_joining(data_dict[Id])\n        labels = train_df.label[train_df.Id == Id].tolist()[0].split(\"|\")\n\n        s, k, l, un_matched = form_labels(sentence=sentence, labels_list = labels)\n\n        if len(s) == 0:\n            un_matched_reviews += 1\n            un_mat.append(un_matched)\n        else: \n            sentences_.append(s)\n            key_.append(k)\n            labels_.append(l)\n            Id_list_.append([Id]*len(l))\n\n\n    print(\"Total unmatched keywords:\", un_matched_reviews)\n    sentences = [item for sublist in sentences_ for item in sublist]\n    final_labels = [item for sublist in labels_ for item in sublist]\n    keywords = [item for sublist in key_ for item in sublist]\n    Id_list = [item for sublist in Id_list_ for item in sublist]\n    \n    return sentences, final_labels, keywords, Id_list\n","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:20:51.423766Z","iopub.execute_input":"2021-06-24T16:20:51.424120Z","iopub.status.idle":"2021-06-24T16:20:51.447522Z","shell.execute_reply.started":"2021-06-24T16:20:51.424087Z","shell.execute_reply":"2021-06-24T16:20:51.446591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import RobertaForTokenClassification","metadata":{"execution":{"iopub.status.busy":"2021-06-24T15:30:53.26883Z","iopub.execute_input":"2021-06-24T15:30:53.269164Z","iopub.status.idle":"2021-06-24T15:30:53.273407Z","shell.execute_reply.started":"2021-06-24T15:30:53.269133Z","shell.execute_reply":"2021-06-24T15:30:53.272534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\ntrain_sentences, train_labels, train_keywords, train_Id_list = labelling(dataset = train_df, data_dict=train_data)\nprint(f\" train sentences: {len(train_sentences)}, train label: {len(train_labels)}, train keywords: {len(train_keywords)}, train_id list: {len(train_Id_list)}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:20:55.141804Z","iopub.execute_input":"2021-06-24T16:20:55.142151Z","iopub.status.idle":"2021-06-24T16:47:42.889939Z","shell.execute_reply.started":"2021-06-24T16:20:55.142121Z","shell.execute_reply":"2021-06-24T16:47:42.889097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove duplicates\nunique_df=pd.DataFrame( {'id':train_Id_list, \n                          'train_sentences': train_sentences, \n                          'kword': train_keywords, \n                          'label':train_labels})\nunique_df.label = unique_df.label.astype('str')\nunique_df.kword = unique_df.kword.astype('str')\nunique_df['sent_len'] = unique_df.train_sentences.apply(lambda x : len(x.split(\" \")))\nunique_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:14.029700Z","iopub.execute_input":"2021-06-24T16:50:14.030037Z","iopub.status.idle":"2021-06-24T16:50:14.525260Z","shell.execute_reply.started":"2021-06-24T16:50:14.030007Z","shell.execute_reply":"2021-06-24T16:50:14.524295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete duplicates \nunique_df = unique_df.drop_duplicates()\n#select the validation sample\nunique_df = unique_df.sample(int(unique_df.shape[0]*0.05)).reset_index(drop=True)\nprint(unique_df.shape)\nnp.random.seed(100)\ntrain_df, valid_df = train_test_split(unique_df, test_size=0.2)\n\ntrain_df = train_df.reset_index(drop=True)\nvalid_df = valid_df.reset_index(drop=True)\n\nprint(train_df.shape, valid_df.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:17.588785Z","iopub.execute_input":"2021-06-24T16:50:17.589126Z","iopub.status.idle":"2021-06-24T16:50:17.691031Z","shell.execute_reply.started":"2021-06-24T16:50:17.589094Z","shell.execute_reply":"2021-06-24T16:50:17.690058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataFrame to list \ntags_2_idx = {'O': 0 , 'B': 1, 'P': 2} # 'P' means padding. \n\ndef dataset_2_list(df):\n    id_list = df.id.values.tolist()\n    sentences_list = df.train_sentences.values.tolist()\n    keywords_list = df.kword.apply(lambda x : eval(x)).values.tolist()\n    \n    labels_list = df.label.apply(lambda x : eval(x)).values.tolist()    \n    labels_list = [list(map(tags_2_idx.get, lab)) for lab in labels_list]\n    \n    return id_list, sentences_list, keywords_list, labels_list\n\nfinal_train_id_list, final_train_sentences, final_train_keywords, final_train_labels = dataset_2_list(df=train_df)\nfinal_valid_id_list, final_valid_sentences, final_valid_keywords, final_valid_labels = dataset_2_list(df=valid_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:19.464546Z","iopub.execute_input":"2021-06-24T16:50:19.464861Z","iopub.status.idle":"2021-06-24T16:50:19.613228Z","shell.execute_reply.started":"2021-06-24T16:50:19.464829Z","shell.execute_reply":"2021-06-24T16:50:19.612396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df\ndel unique_df","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:22.900818Z","iopub.execute_input":"2021-06-24T16:50:22.901175Z","iopub.status.idle":"2021-06-24T16:50:22.905237Z","shell.execute_reply.started":"2021-06-24T16:50:22.901136Z","shell.execute_reply":"2021-06-24T16:50:22.904191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forming the input\nclass form_input():\n    def __init__(self, ID, sentence, kword, label, data_type='test'):\n        self.id = ID\n        self.sentence = sentence\n        self.kword = kword\n        self.label = label\n        self.max_length = config['MAX_LEN']\n        self.tokenizer = config['tokenizer']\n        self.data_type = data_type\n    \n    def __len__(self):\n        return len(self.sentence)\n    \n    def __getitem__(self, item):\n        toks = config['tokenizer'].tokenize(\" \" + self.sentence[item])\n        label = self.label[item]\n\n        if len(toks)>self.max_length:\n            toks = toks[:self.max_length]\n            label = label[:self.max_length]\n                \n        ########################################\n        # Forming the inputs\n        ids = config['tokenizer'].convert_tokens_to_ids(toks)\n        tok_type_id = [0] * len(ids)\n        att_mask = [1] * len(ids)\n        \n        # Padding\n        pad_len = self.max_length - len(ids)        \n        ids = ids + [2] * pad_len\n        tok_type_id = tok_type_id + [0] * pad_len\n        att_mask = att_mask + [0] * pad_len\n        \n        ########################################            \n        # Forming the label\n        if self.data_type !='test':\n            label = label + [2]*pad_len\n        else:\n            label = 1\n        \n        ########################################\n                \n        return {'pub_id': self.id[item],\n                #'item': item,\n                #'sentence': self.sentence[item],\n                #'kword' : self.kword[item],\n                'ids': torch.tensor(ids, dtype = torch.long),\n                'tok_type_id': torch.tensor(tok_type_id, dtype = torch.long),\n                'att_mask': torch.tensor(att_mask, dtype = torch.long),\n                'target': torch.tensor(label, dtype = torch.long)\n               }","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:24.190187Z","iopub.execute_input":"2021-06-24T16:50:24.190499Z","iopub.status.idle":"2021-06-24T16:50:24.200332Z","shell.execute_reply.started":"2021-06-24T16:50:24.190469Z","shell.execute_reply":"2021-06-24T16:50:24.199351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataLoader \ntrain_prod_input = form_input(ID=final_train_id_list, \n                              sentence=final_train_sentences, \n                              kword=final_train_keywords, \n                              label=final_train_labels, \n                              data_type='train')\n\nvalid_prod_input = form_input(ID=final_valid_id_list, \n                              sentence=final_valid_sentences, \n                              kword=final_valid_keywords, \n                              label=final_valid_labels, \n                              data_type='valid')\n\ntrain_prod_input_data_loader = DataLoader(train_prod_input, \n                                          batch_size= config['batch_size'], \n                                          shuffle=True)\n\nvalid_prod_input_data_loader = DataLoader(valid_prod_input, \n                                          batch_size= config['batch_size'], \n                                          shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:26.501920Z","iopub.execute_input":"2021-06-24T16:50:26.502240Z","iopub.status.idle":"2021-06-24T16:50:26.507659Z","shell.execute_reply.started":"2021-06-24T16:50:26.502210Z","shell.execute_reply":"2021-06-24T16:50:26.506826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def setting_seed(seed_no=100):\n    random.seed(seed_no)\n    np.random.seed(seed_no)\n    torch.manual_seed(seed_no)\n    torch.cuda.manual_seed_all(seed_no) ","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:28.588702Z","iopub.execute_input":"2021-06-24T16:50:28.589041Z","iopub.status.idle":"2021-06-24T16:50:28.595730Z","shell.execute_reply.started":"2021-06-24T16:50:28.589008Z","shell.execute_reply":"2021-06-24T16:50:28.594633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training function\ndef train_fn(data_loader, model, optimizer):\n    '''\n    Function to train the model\n    '''\n    setting_seed(seed_no=100)\n    train_loss = 0\n    for index, dataset in enumerate(tqdm(data_loader, total = len(data_loader))):\n        batch_input_ids = dataset['ids'].to(config['device'], dtype = torch.long)\n        batch_att_mask = dataset['att_mask'].to(config['device'], dtype = torch.long)\n        batch_tok_type_id = dataset['tok_type_id'].to(config['device'], dtype = torch.long)\n        batch_target = dataset['target'].to(config['device'], dtype = torch.long)\n                \n        output = model(batch_input_ids, \n                       token_type_ids=None,\n                       attention_mask=batch_att_mask,\n                       labels=batch_target)\n        \n        step_loss = output[0]\n        prediction = output[1]\n        \n        step_loss.sum().backward()\n        optimizer.step()        \n        train_loss += step_loss\n        optimizer.zero_grad()\n        \n    return train_loss.sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:31.189501Z","iopub.execute_input":"2021-06-24T16:50:31.192725Z","iopub.status.idle":"2021-06-24T16:50:31.204706Z","shell.execute_reply.started":"2021-06-24T16:50:31.192676Z","shell.execute_reply":"2021-06-24T16:50:31.203843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\ndef eval_fn(data_loader, model):\n    '''\n    Functiont to evaluate the model on each epoch. \n    We can also use Jaccard metric to see the performance on each epoch.\n    '''\n    setting_seed(seed_no=100)\n    model.eval()\n    \n    eval_loss = 0\n    predictions = np.array([], dtype = np.int64).reshape(0, config['MAX_LEN'])\n    true_labels = np.array([], dtype = np.int64).reshape(0, config['MAX_LEN'])\n    \n    with torch.no_grad():\n        for index, dataset in enumerate(tqdm(data_loader, total = len(data_loader))):\n            batch_input_ids = dataset['ids'].to(config['device'], dtype = torch.long)\n            batch_att_mask = dataset['att_mask'].to(config['device'], dtype = torch.long)\n            batch_tok_type_id = dataset['tok_type_id'].to(config['device'], dtype = torch.long)\n            batch_target = dataset['target'].to(config['device'], dtype = torch.long)\n\n            output = model(batch_input_ids, \n                           token_type_ids=None,\n                           attention_mask=batch_att_mask,\n                           labels=batch_target)\n\n            step_loss = output[0]\n            eval_prediction = output[1]\n\n            eval_loss += step_loss\n            \n            eval_prediction = np.argmax(eval_prediction.detach().to('cpu').numpy(), axis = 2)\n            actual = batch_target.to('cpu').numpy()\n            \n            predictions = np.concatenate((predictions, eval_prediction), axis = 0)\n            true_labels = np.concatenate((true_labels, actual), axis = 0)\n            \n    return eval_loss.sum(), predictions, true_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:50:33.030138Z","iopub.execute_input":"2021-06-24T16:50:33.030453Z","iopub.status.idle":"2021-06-24T16:50:33.039776Z","shell.execute_reply.started":"2021-06-24T16:50:33.030424Z","shell.execute_reply":"2021-06-24T16:50:33.038948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_engine(epoch, train_data, valid_data):\n    setting_seed(seed_no=100)\n    model = RobertaForTokenClassification.from_pretrained('../input/huggingface-roberta/roberta-base',  num_labels = len(tags_2_idx))\n    model =roberta_tokenizer\n    model = nn.DataParallel(model)\n    #model = model.to(config['device'])\n    \n    params = model.parameters()\n    optimizer = torch.optim.Adam(params, lr= 3e-5)\n    \n    best_eval_loss = 1000000\n    for i in range(epoch):\n        train_loss = train_fn(data_loader = train_data, \n                              model=model, \n                              optimizer=optimizer)\n        eval_loss, eval_predictions, true_labels = eval_fn(data_loader = valid_data, \n                                                           model=model)\n        \n        print(f\"Epoch {i} , Train loss: {train_loss}, Eval loss: {eval_loss}\")\n\n        if eval_loss < best_eval_loss:\n            best_eval_loss = eval_loss           \n            \n            print(\"Saving the model\")\n            torch.save(model.state_dict(), config['model_name'])\n            \n    return model, eval_predictions, true_labels ","metadata":{"execution":{"iopub.status.busy":"2021-06-24T17:42:07.230346Z","iopub.execute_input":"2021-06-24T17:42:07.230680Z","iopub.status.idle":"2021-06-24T17:42:07.238087Z","shell.execute_reply.started":"2021-06-24T17:42:07.230631Z","shell.execute_reply":"2021-06-24T17:42:07.237123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, val_predictions, val_true_labels = train_engine(epoch=config['Epoch'],train_data=train_prod_input_data_loader, valid_data=valid_prod_input_data_loader)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T17:26:26.222034Z","iopub.execute_input":"2021-06-24T17:26:26.222404Z","iopub.status.idle":"2021-06-24T17:26:36.542409Z","shell.execute_reply.started":"2021-06-24T17:26:26.222370Z","shell.execute_reply":"2021-06-24T17:26:36.540805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_test_json(test_data_folder):\n    '''\n    This function reads all the json input files and return a dictionary containing the id as the key\n    and all the contents of the json as values\n    '''\n\n    test_text_data = {}\n    total_files = len(glob.glob(test_data_folder))\n    \n    for i, test_json_loc in enumerate(glob.glob(test_data_folder)):\n        filename = test_json_loc.split(\"/\")[-1][:-5]\n\n        with open(test_json_loc, 'r') as f:\n            test_text_data[filename] = json.load(f)\n\n    print(\"All files read\")\n    return test_text_data","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:10:54.344692Z","iopub.execute_input":"2021-06-22T10:10:54.345159Z","iopub.status.idle":"2021-06-22T10:10:54.354982Z","shell.execute_reply.started":"2021-06-22T10:10:54.345105Z","shell.execute_reply":"2021-06-22T10:10:54.353384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_dict = read_test_json(test_data_folder=config['test_path'])","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:10:58.776262Z","iopub.execute_input":"2021-06-22T10:10:58.776781Z","iopub.status.idle":"2021-06-22T10:10:58.820132Z","shell.execute_reply.started":"2021-06-22T10:10:58.776734Z","shell.execute_reply":"2021-06-22T10:10:58.818731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction\ndef prediction_fn(tokenized_sub_sentence):\n\n    tkns = tokenized_sub_sentence\n    indexed_tokens = config['tokenizer'].convert_tokens_to_ids(tkns)\n    segments_ids = [0] * len(indexed_tokens)\n\n    tokens_tensor = torch.tensor([indexed_tokens]).to(config['device'])\n    segments_tensors = torch.tensor([segments_ids]).to(config['device'])\n    \n    setting_seed(seed_no=100)\n    model.eval()\n    with torch.no_grad():\n        logit = model(tokens_tensor, \n                      token_type_ids=None,\n                      attention_mask=segments_tensors)\n\n        logit_new = logit[0].argmax(2).detach().cpu().numpy().tolist()\n        prediction = logit_new[0]\n\n#         print(tkns)\n#         print(logit_new)\n#         print(prediction)\n        \n        kword = ''\n        kword_list = []\n\n        for k, j in enumerate(prediction):\n            if (len(prediction)>1):\n\n                if (j!=0) & (k==0):\n                    #if it's the first word in the first position\n                    #print('At begin first word')\n                    begin = tkns[k]\n                    kword = begin\n\n                elif (j!=0) & (k>=1) & (prediction[k-1]==0):\n                    #begin word is in the middle of the sentence\n                    begin = tkns[k]\n                    previous = tkns[k-1]\n\n                    if not begin.startswith('Ġ'):\n                        kword = previous + begin\n                    else:\n                        kword = begin\n\n                    if k == (len(prediction) - 1):\n                        #print('begin and end word is the last word of the sentence')\n                        kword_list.append(kword.rstrip().lstrip().replace('Ġ', ''))\n\n                elif (j!=0) & (k>=1) & (prediction[k-1]!=0):\n                    # intermediate word of the same keyword\n                    inter = tkns[k]\n\n                    if not inter.startswith('Ġ'):\n                        kword = kword + \"\" + inter\n                    else:\n                        kword = kword + \" \" + inter\n\n\n                    if k == (len(prediction) - 1):\n                        #print('begin and end')\n                        kword_list.append(kword.rstrip().lstrip().replace('Ġ', ''))\n\n                elif (j==0) & (k>=1) & (prediction[k-1] !=0):\n                    # End of a keywords but not end of sentence.\n                    kword_list.append(kword.rstrip().lstrip().replace('Ġ', ''))\n                    kword = ''\n                    inter = ''\n            else:\n                if (j!=0):\n                    begin = tkns[k]\n                    kword = begin\n                    kword_list.append(kword.rstrip().lstrip().replace('Ġ', ''))\n#         print(kword_list)\n#         print(\"\")\n    return kword_list","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:11:01.014792Z","iopub.execute_input":"2021-06-22T10:11:01.015209Z","iopub.status.idle":"2021-06-22T10:11:01.033831Z","shell.execute_reply.started":"2021-06-22T10:11:01.015169Z","shell.execute_reply":"2021-06-22T10:11:01.032516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def long_sent_split(text):\n    sent_split = text.split(\" \")\n\n    start = 0\n    end = len(sent_split)\n    max_length = 64\n\n    final_sent_split = []\n    for i in range(start, end, max_length):\n        temp = sent_split[i: (i + max_length)]\n        final_sent_split.append(\" \".join(i for i in temp))\n    return final_sent_split","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:11:04.176928Z","iopub.execute_input":"2021-06-22T10:11:04.17732Z","iopub.status.idle":"2021-06-22T10:11:04.187429Z","shell.execute_reply.started":"2021-06-22T10:11:04.177262Z","shell.execute_reply":"2021-06-22T10:11:04.186178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(data_dict):\n    \n    results = {}\n\n    for i, Id in enumerate(data_dict.keys()):\n        current_id_predictions = []\n    \n        print(Id)\n        sentences = data_joining(data_dict[Id])\n        sentence_tokens = sent_tokenize(sentences)\n        \n        for sub_sentence in sentence_tokens:\n            cleaned_sub_sentence = clean_text(sub_sentence)\n        \n            # Tokenize the sentence\n            tokenized_sub_sentence = config['tokenizer'].tokenize(\" \" + cleaned_sub_sentence)\n            \n            if len(tokenized_sub_sentence) == 0:\n                # If the tokenized sentence are empty\n                sub_sentence_prediction_kword_list = []\n                \n            elif len(tokenized_sub_sentence) <= 512:\n                # If the tokenized sentence are less than 512\n                sub_sentence_prediction_kword_list = prediction_fn(tokenized_sub_sentence)\n\n            else:\n                # If the tokenized sentence are >512 which is long sentences\n                long_sent_kword_list = []\n                \n                tokenized_sub_sentence_tok_split = long_sent_split(text = tokenized_sub_sentence)\n                for i, sent_tok in enumerate(tokenized_sub_sentence_tok_split):\n                    if len(sent) != 0:\n                        kword_list = prediction_fn(sent_tok)\n                        long_sent_kword_list.append(kword_list)\n                flat_long_sent_kword = [item for sublist in long_sent_kword_list for item in sublist]\n                sub_sentence_prediction_kword_list = flat_long_sent_kword\n                            \n            if len(sub_sentence_prediction_kword_list) !=0:\n                current_id_predictions = current_id_predictions + sub_sentence_prediction_kword_list\n\n        results[Id] = list(set(current_id_predictions))\n                \n    print(\"All predictions completed\")\n    \n    return results","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:11:06.509227Z","iopub.execute_input":"2021-06-22T10:11:06.509852Z","iopub.status.idle":"2021-06-22T10:11:06.521054Z","shell.execute_reply.started":"2021-06-22T10:11:06.509807Z","shell.execute_reply":"2021-06-22T10:11:06.519809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults = get_predictions(data_dict = test_data_dict)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:11:08.916134Z","iopub.execute_input":"2021-06-22T10:11:08.916911Z","iopub.status.idle":"2021-06-22T10:11:52.579649Z","shell.execute_reply.started":"2021-06-22T10:11:08.916863Z","shell.execute_reply":"2021-06-22T10:11:52.577524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame({'Id': list(results.keys()),\n                       'PredictionString': list(results.values())})\nsub_df.PredictionString = sub_df.PredictionString.apply(lambda x : \"|\".join(x))\nsub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:12:25.857093Z","iopub.execute_input":"2021-06-22T10:12:25.857595Z","iopub.status.idle":"2021-06-22T10:12:25.869714Z","shell.execute_reply.started":"2021-06-22T10:12:25.857547Z","shell.execute_reply":"2021-06-22T10:12:25.86807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Save predictions\n\n#sample_submission['PredictionString'] = sub_df\n#sample_submission[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\n","metadata":{},"execution_count":null,"outputs":[]}]}