{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import packages","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nfrom tqdm import tqdm\nfrom transformers import pipeline\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom flashtext import KeywordProcessor\nfrom tqdm._tqdm_notebook import tqdm_notebook\ntqdm_notebook.pandas()\nfrom nltk import tokenize\nfrom transformers import AdamW\nfrom collections import Counter\nfrom transformers import AutoTokenizer, AutoModel\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\nfrom transformers import AutoModelForTokenClassification\nfrom transformers import AutoTokenizer, AutoConfig, TrainingArguments, Trainer, EvalPrediction\nimport sys\nimport time\nimport gc\nimport pickle\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.113338Z","iopub.execute_input":"2021-06-13T13:20:00.113662Z","iopub.status.idle":"2021-06-13T13:20:00.121299Z","shell.execute_reply.started":"2021-06-13T13:20:00.113628Z","shell.execute_reply":"2021-06-13T13:20:00.120199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install GPUtil\n# import torch\n# from GPUtil import showUtilization as gpu_usage\n# from numba import cuda\n\n# def free_gpu_cache():\n#     print(\"Initial GPU Usage\")\n#     gpu_usage()                             \n\n#     torch.cuda.empty_cache()\n\n#     cuda.select_device(0)\n#     cuda.close()\n#     cuda.select_device(0)\n\n#     print(\"GPU Usage after emptying the cache\")\n#     gpu_usage()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.122959Z","iopub.execute_input":"2021-06-13T13:20:00.123773Z","iopub.status.idle":"2021-06-13T13:20:00.133470Z","shell.execute_reply.started":"2021-06-13T13:20:00.123736Z","shell.execute_reply":"2021-06-13T13:20:00.132275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.135604Z","iopub.execute_input":"2021-06-13T13:20:00.136206Z","iopub.status.idle":"2021-06-13T13:20:00.843227Z","shell.execute_reply.started":"2021-06-13T13:20:00.136164Z","shell.execute_reply":"2021-06-13T13:20:00.842173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load input files","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsubmission_file = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.844928Z","iopub.execute_input":"2021-06-13T13:20:00.845248Z","iopub.status.idle":"2021-06-13T13:20:00.955270Z","shell.execute_reply.started":"2021-06-13T13:20:00.845215Z","shell.execute_reply":"2021-06-13T13:20:00.954436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"markdown","source":"### Train data samples","metadata":{}},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.956581Z","iopub.execute_input":"2021-06-13T13:20:00.956911Z","iopub.status.idle":"2021-06-13T13:20:00.962432Z","shell.execute_reply.started":"2021-06-13T13:20:00.956876Z","shell.execute_reply":"2021-06-13T13:20:00.961520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.963811Z","iopub.execute_input":"2021-06-13T13:20:00.964360Z","iopub.status.idle":"2021-06-13T13:20:00.972639Z","shell.execute_reply.started":"2021-06-13T13:20:00.964323Z","shell.execute_reply":"2021-06-13T13:20:00.971836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train data shape:', train_data.shape)\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:00.973982Z","iopub.execute_input":"2021-06-13T13:20:00.974655Z","iopub.status.idle":"2021-06-13T13:20:01.001921Z","shell.execute_reply.started":"2021-06-13T13:20:00.974615Z","shell.execute_reply":"2021-06-13T13:20:01.000918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = '../input/coleridgeinitiative-show-us-the-data/train/0008656f-0ba2-4632-8602-3017b44c2e90.json'\nfile_id = file_path.split('/')[-1].split('.json')[0]\nwith open(file_path) as json_file:\n    data = json.load(json_file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.004976Z","iopub.execute_input":"2021-06-13T13:20:01.005343Z","iopub.status.idle":"2021-06-13T13:20:01.016207Z","shell.execute_reply.started":"2021-06-13T13:20:01.005307Z","shell.execute_reply":"2021-06-13T13:20:01.015247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of elements:', len(data))\ncombined_text = ' '.join([x['text'] for x in data])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.019512Z","iopub.execute_input":"2021-06-13T13:20:01.020123Z","iopub.status.idle":"2021-06-13T13:20:01.025786Z","shell.execute_reply.started":"2021-06-13T13:20:01.020073Z","shell.execute_reply":"2021-06-13T13:20:01.024526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(combined_text), combined_text[0:1000]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.027697Z","iopub.execute_input":"2021-06-13T13:20:01.028205Z","iopub.status.idle":"2021-06-13T13:20:01.036199Z","shell.execute_reply.started":"2021-06-13T13:20:01.028165Z","shell.execute_reply":"2021-06-13T13:20:01.035105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Dataset label:', train_data[train_data['Id']==file_id]['dataset_label'].iloc[0],'\\n')\nstart_index = combined_text.find(train_data[train_data['Id']==file_id]['dataset_label'].iloc[0])\nprint('Text for the dataset label\\n',combined_text[start_index-100:start_index+100])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.038004Z","iopub.execute_input":"2021-06-13T13:20:01.038552Z","iopub.status.idle":"2021-06-13T13:20:01.058108Z","shell.execute_reply.started":"2021-06-13T13:20:01.038510Z","shell.execute_reply":"2021-06-13T13:20:01.057127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[train_data['Id']==file_id]['pub_title'].iloc[0],train_data[train_data['Id']==file_id]['dataset_title'].iloc[0],train_data[train_data['Id']==file_id]['dataset_label'].iloc[0],train_data[train_data['Id']==file_id]['cleaned_label'].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.059464Z","iopub.execute_input":"2021-06-13T13:20:01.059846Z","iopub.status.idle":"2021-06-13T13:20:01.082700Z","shell.execute_reply.started":"2021-06-13T13:20:01.059807Z","shell.execute_reply":"2021-06-13T13:20:01.081639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train data shape:', train_data.shape)\nprint('Number of unique IDs in train data:', train_data['Id'].nunique())\nprint('Number of unique pub_titles:', train_data['pub_title'].nunique())\nprint('Number of unique dataset titles:', train_data['dataset_title'].nunique())\nprint('Number of unique dataset labels:', train_data['dataset_label'].nunique())\nprint('Number of unique cleaned labels:', train_data['cleaned_label'].nunique())","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.084179Z","iopub.execute_input":"2021-06-13T13:20:01.084569Z","iopub.status.idle":"2021-06-13T13:20:01.113669Z","shell.execute_reply.started":"2021-06-13T13:20:01.084531Z","shell.execute_reply":"2021-06-13T13:20:01.112756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(submission_file.shape,submission_file['Id'].nunique())\nsubmission_file","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.115140Z","iopub.execute_input":"2021-06-13T13:20:01.115533Z","iopub.status.idle":"2021-06-13T13:20:01.129857Z","shell.execute_reply.started":"2021-06-13T13:20:01.115477Z","shell.execute_reply":"2021-06-13T13:20:01.128733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Identifying problem as NER","metadata":{}},{"cell_type":"code","source":"# Check if problem is NER (named entity recognition) - dataset label should be somewhere in text for each row\n\nnumber_of_elements_in_text = pd.DataFrame(data = None, columns = ['Train Id','Number of elements','Total document length (words)','Total document length (char)','start_index in text for dataset label','start_index in section_title for dataset label'])\nall_train_ids = train_data['Id'].unique().tolist()\ncount = 0\nfor i in tqdm(all_train_ids):    \n    file_path = '../input/coleridgeinitiative-show-us-the-data/train//' + i + '.json'\n    file_id = file_path.split('/')[-1].split('.json')[0]\n    with open(file_path) as json_file:\n        data = json.load(json_file)\n        \n    combined_section_title = ' '.join([x['section_title'].strip().lower() for x in data])\n    combined_text = ' '.join([x['text'].strip().lower() for x in data])\n    \n    try:\n        start_index_text = combined_text.find((train_data[train_data['Id']==file_id]['dataset_label'].iloc[0]).lower().strip())\n    except:\n        start_index_text = np.NaN\n        \n    try:\n        start_index_section_title = combined_section_title.find((train_data[train_data['Id']==file_id]['dataset_label'].iloc[0]).lower().strip())\n    except:\n        start_index_section_title = np.NaN\n    \n    \n    number_of_elements_in_text.loc[count] = i, len(data), len(combined_text.strip().split(' ')), len(combined_text.strip()), start_index_text, start_index_section_title\n    count = count + 1    ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:20:01.131218Z","iopub.execute_input":"2021-06-13T13:20:01.131804Z","iopub.status.idle":"2021-06-13T13:23:32.023561Z","shell.execute_reply.started":"2021-06-13T13:20:01.131759Z","shell.execute_reply":"2021-06-13T13:23:32.022735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_train_ids)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.024789Z","iopub.execute_input":"2021-06-13T13:23:32.025130Z","iopub.status.idle":"2021-06-13T13:23:32.032778Z","shell.execute_reply.started":"2021-06-13T13:23:32.025093Z","shell.execute_reply":"2021-06-13T13:23:32.031696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(number_of_elements_in_text.shape)\nnumber_of_elements_in_text.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.034232Z","iopub.execute_input":"2021-06-13T13:23:32.034751Z","iopub.status.idle":"2021-06-13T13:23:32.055080Z","shell.execute_reply.started":"2021-06-13T13:23:32.034710Z","shell.execute_reply":"2021-06-13T13:23:32.054136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Percentage of documents with dataset label in text:', number_of_elements_in_text[(number_of_elements_in_text['start_index in text for dataset label']!=-1)].shape[0]/number_of_elements_in_text.shape[0] * 100,'%')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.056936Z","iopub.execute_input":"2021-06-13T13:23:32.057533Z","iopub.status.idle":"2021-06-13T13:23:32.069916Z","shell.execute_reply.started":"2021-06-13T13:23:32.057489Z","shell.execute_reply":"2021-06-13T13:23:32.068826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_elements_in_text[(number_of_elements_in_text['start_index in text for dataset label']==-1)].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.071590Z","iopub.execute_input":"2021-06-13T13:23:32.072009Z","iopub.status.idle":"2021-06-13T13:23:32.081762Z","shell.execute_reply.started":"2021-06-13T13:23:32.071970Z","shell.execute_reply":"2021-06-13T13:23:32.080881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All dataset labels occur in the text\nnumber_of_elements_in_text[(number_of_elements_in_text['start_index in text for dataset label']==-1) & (number_of_elements_in_text['start_index in section_title for dataset label']!=-1)].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.083197Z","iopub.execute_input":"2021-06-13T13:23:32.083661Z","iopub.status.idle":"2021-06-13T13:23:32.096905Z","shell.execute_reply.started":"2021-06-13T13:23:32.083622Z","shell.execute_reply":"2021-06-13T13:23:32.096147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Very few dataset labels occur in section_title\nnumber_of_elements_in_text[(number_of_elements_in_text['start_index in section_title for dataset label']!=-1)].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.099533Z","iopub.execute_input":"2021-06-13T13:23:32.099825Z","iopub.status.idle":"2021-06-13T13:23:32.110609Z","shell.execute_reply.started":"2021-06-13T13:23:32.099800Z","shell.execute_reply":"2021-06-13T13:23:32.109687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_elements_in_text[number_of_elements_in_text['start_index in text for dataset label']==-1].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.112793Z","iopub.execute_input":"2021-06-13T13:23:32.113497Z","iopub.status.idle":"2021-06-13T13:23:32.129499Z","shell.execute_reply.started":"2021-06-13T13:23:32.113463Z","shell.execute_reply":"2021-06-13T13:23:32.128498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = '../input/coleridgeinitiative-show-us-the-data/train/c9050bc3-2551-4f41-9f40-2851fc705c3c.json'\nfile_id = file_path.split('/')[-1].split('.json')[0]\nwith open(file_path) as json_file:\n    data = json.load(json_file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.135323Z","iopub.execute_input":"2021-06-13T13:23:32.135581Z","iopub.status.idle":"2021-06-13T13:23:32.142289Z","shell.execute_reply.started":"2021-06-13T13:23:32.135553Z","shell.execute_reply":"2021-06-13T13:23:32.141473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Full forms and rare dataset names do not occur in text nor in section_title\ntrain_data[train_data['Id']==file_id]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.145301Z","iopub.execute_input":"2021-06-13T13:23:32.145716Z","iopub.status.idle":"2021-06-13T13:23:32.160177Z","shell.execute_reply.started":"2021-06-13T13:23:32.145678Z","shell.execute_reply":"2021-06-13T13:23:32.159093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_section_title = ' '.join([x['section_title'].strip().lower() for x in data])\ncombined_text = ' '.join([x['text'].strip().lower() for x in data])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.161632Z","iopub.execute_input":"2021-06-13T13:23:32.162001Z","iopub.status.idle":"2021-06-13T13:23:32.169618Z","shell.execute_reply.started":"2021-06-13T13:23:32.161964Z","shell.execute_reply":"2021-06-13T13:23:32.168591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_data[train_data['Id']==file_id]['dataset_label'].iloc[0]).strip().lower() in combined_section_title,(train_data[train_data['Id']==file_id]['dataset_label'].iloc[0]).strip().lower() in combined_text","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.170656Z","iopub.execute_input":"2021-06-13T13:23:32.170921Z","iopub.status.idle":"2021-06-13T13:23:32.187297Z","shell.execute_reply.started":"2021-06-13T13:23:32.170897Z","shell.execute_reply":"2021-06-13T13:23:32.186283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[train_data['Id'].isin(number_of_elements_in_text[number_of_elements_in_text['start_index in text for dataset label']==-1]['Train Id'].unique().tolist())]['dataset_label'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.188819Z","iopub.execute_input":"2021-06-13T13:23:32.189222Z","iopub.status.idle":"2021-06-13T13:23:32.206543Z","shell.execute_reply.started":"2021-06-13T13:23:32.189174Z","shell.execute_reply":"2021-06-13T13:23:32.205753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of elements in each document","metadata":{}},{"cell_type":"code","source":"df = number_of_elements_in_text[['Train Id','Number of elements']].drop_duplicates()\nprint(df.shape)\nfig = px.histogram(df, x='Number of elements',nbins = 200)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:32.208796Z","iopub.execute_input":"2021-06-13T13:23:32.209160Z","iopub.status.idle":"2021-06-13T13:23:33.136205Z","shell.execute_reply.started":"2021-06-13T13:23:32.209127Z","shell.execute_reply":"2021-06-13T13:23:33.135398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Length of documents","metadata":{}},{"cell_type":"code","source":"df = number_of_elements_in_text[['Train Id','Total document length (words)']].drop_duplicates()\nprint(df.shape)\nprint('Maximum length:',df['Total document length (words)'].max())\nfig = px.histogram(df, x='Total document length (words)',nbins = 1000)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.137234Z","iopub.execute_input":"2021-06-13T13:23:33.137484Z","iopub.status.idle":"2021-06-13T13:23:33.288600Z","shell.execute_reply.started":"2021-06-13T13:23:33.137458Z","shell.execute_reply":"2021-06-13T13:23:33.287711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of documents of cleaned labels","metadata":{}},{"cell_type":"code","source":"train_data_cl_label_summary = train_data.groupby(['cleaned_label']).agg({'Id':'nunique'}).reset_index().rename(columns = {'Id':'Number of documents'}).sort_values(by = 'Number of documents', ascending = False)\ntrain_data_cl_label_summary.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.289669Z","iopub.execute_input":"2021-06-13T13:23:33.289988Z","iopub.status.idle":"2021-06-13T13:23:33.314175Z","shell.execute_reply.started":"2021-06-13T13:23:33.289955Z","shell.execute_reply":"2021-06-13T13:23:33.313216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_for_plot = train_data_cl_label_summary.head(20).sort_values(by = 'Number of documents')\nplt.barh(data_for_plot.head(20)['cleaned_label'],data_for_plot.head(20)['Number of documents'])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.315575Z","iopub.execute_input":"2021-06-13T13:23:33.315954Z","iopub.status.idle":"2021-06-13T13:23:33.586805Z","shell.execute_reply.started":"2021-06-13T13:23:33.315916Z","shell.execute_reply":"2021-06-13T13:23:33.585888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Length of cleaned labels","metadata":{}},{"cell_type":"code","source":"train_data['cleaned_label_length'] = train_data['cleaned_label'].apply(lambda x: len(x.strip().split(' ')))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.588272Z","iopub.execute_input":"2021-06-13T13:23:33.588615Z","iopub.status.idle":"2021-06-13T13:23:33.611500Z","shell.execute_reply.started":"2021-06-13T13:23:33.588579Z","shell.execute_reply":"2021-06-13T13:23:33.610612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_data[['cleaned_label']].drop_duplicates().reset_index(drop = True)\ndata['cleaned_label_length'] = data['cleaned_label'].apply(lambda x: len(x.strip().split(' ')))\ntrain_data_cl_label_len_summary = data.groupby(['cleaned_label_length']).agg({'cleaned_label':'nunique'}).reset_index().rename(columns = {'cleaned_label':'Number of cleaned labels'}).sort_values(by = 'Number of cleaned labels', ascending = False)\ntrain_data_cl_label_len_summary.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.613918Z","iopub.execute_input":"2021-06-13T13:23:33.614284Z","iopub.status.idle":"2021-06-13T13:23:33.638573Z","shell.execute_reply.started":"2021-06-13T13:23:33.614229Z","shell.execute_reply":"2021-06-13T13:23:33.637873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_for_plot = train_data_cl_label_len_summary.sort_values(by = 'Number of cleaned labels')\nplt.barh(data_for_plot['cleaned_label_length'],data_for_plot['Number of cleaned labels'])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.639797Z","iopub.execute_input":"2021-06-13T13:23:33.640159Z","iopub.status.idle":"2021-06-13T13:23:33.778774Z","shell.execute_reply.started":"2021-06-13T13:23:33.640124Z","shell.execute_reply":"2021-06-13T13:23:33.777876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Minimum length:', train_data_cl_label_len_summary['cleaned_label_length'].min())\nprint('Maximum length:', train_data_cl_label_len_summary['cleaned_label_length'].max())","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.780050Z","iopub.execute_input":"2021-06-13T13:23:33.780392Z","iopub.status.idle":"2021-06-13T13:23:33.787129Z","shell.execute_reply.started":"2021-06-13T13:23:33.780358Z","shell.execute_reply":"2021-06-13T13:23:33.785956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NER Model","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.788765Z","iopub.execute_input":"2021-06-13T13:23:33.789492Z","iopub.status.idle":"2021-06-13T13:23:33.804524Z","shell.execute_reply.started":"2021-06-13T13:23:33.789391Z","shell.execute_reply":"2021-06-13T13:23:33.803710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_for_model = train_data['dataset_label'].str.strip().str.lower().unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.805794Z","iopub.execute_input":"2021-06-13T13:23:33.806190Z","iopub.status.idle":"2021-06-13T13:23:33.831354Z","shell.execute_reply.started":"2021-06-13T13:23:33.806154Z","shell.execute_reply":"2021-06-13T13:23:33.830462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keywordprocessor = KeywordProcessor()\nkeywordprocessor.add_keywords_from_list(keyword_list=labels_for_model)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.832550Z","iopub.execute_input":"2021-06-13T13:23:33.832849Z","iopub.status.idle":"2021-06-13T13:23:33.838597Z","shell.execute_reply.started":"2021-06-13T13:23:33.832819Z","shell.execute_reply":"2021-06-13T13:23:33.837713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_sentence(x):\n    \"takes in a string and returns tokenized list after special character padded\"\n\n    return [x for x in x.strip().lower().split(\" \") if len(x) > 0]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.839970Z","iopub.execute_input":"2021-06-13T13:23:33.840643Z","iopub.status.idle":"2021-06-13T13:23:33.847385Z","shell.execute_reply.started":"2021-06-13T13:23:33.840607Z","shell.execute_reply":"2021-06-13T13:23:33.846532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_tags(sent, ep):\n    '''\n    Input: sent as a sentence tokenized as list of tokens, ep is list of eparker strings (not tokenized)\n    output: tags \n    '''\n    sent = [x.lower() for x in sent]\n    ep_non_nan = False\n    if isinstance(ep, list):\n        ep_non_nan = True\n        ep = [tokenize_sentence(x.lower()) for x in ep]\n    i = 0\n    tag = []\n    if(ep_non_nan):\n        while (i < len(sent)) and (len(ep) > 0):\n            if (len(ep[0]) == 1) and (ep[0][0] == sent[i]):\n                tag.append(\"B\")\n                i = i + 1\n                ep = ep[1:]\n\n            elif (len(ep[0]) > 1) and (ep[0] == sent[i:i + len(ep[0])]):\n                tag = tag + ['B'] + ['I'] * (len(ep[0]) - 1)\n                i = i + len(ep[0])\n                ep = ep[1:]\n\n            else:\n                tag.append(\"O\")\n                i = i + 1\n\n    tag = tag + ['O'] * (len(sent) - len(tag))\n\n    return tag","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.849958Z","iopub.execute_input":"2021-06-13T13:23:33.850321Z","iopub.status.idle":"2021-06-13T13:23:33.861079Z","shell.execute_reply.started":"2021-06-13T13:23:33.850289Z","shell.execute_reply":"2021-06-13T13:23:33.860095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vocab_sent_tokenize_label(sent_tokenized, token_tag):\n    try:\n        vocab_sent_token = []\n        sent_input_ids = []\n        vocab_token_tag = []\n        token_tag_ids = []\n        for sent_token_, tag_ in zip(sent_tokenized, token_tag):\n            _vocab_sent_token = tokenizer.tokenize(sent_token_)\n            _sent_input_ids = [\n                tokenizer.convert_tokens_to_ids(x) for x in _vocab_sent_token\n            ]\n            _vocab_token_tag = [tag_] * len(_vocab_sent_token)\n            _token_tag_ids = [tag2idx[x] for x in _vocab_token_tag]\n\n            vocab_sent_token.extend(_vocab_sent_token)\n            sent_input_ids.extend(_sent_input_ids)\n            vocab_token_tag.extend(_vocab_token_tag)\n            token_tag_ids.extend(_token_tag_ids)\n        return vocab_sent_token, sent_input_ids, vocab_token_tag, token_tag_ids\n    except Exception as e:\n        print(f\"Error in line no: {sys.exc_info()[2].tb_lineno}\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.862615Z","iopub.execute_input":"2021-06-13T13:23:33.863040Z","iopub.status.idle":"2021-06-13T13:23:33.870686Z","shell.execute_reply.started":"2021-06-13T13:23:33.862982Z","shell.execute_reply":"2021-06-13T13:23:33.869761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sent_tag_tokenization(data):\n    try:\n        model_data_preprocessing = data.copy(deep=True)\n        model_data_preprocessing['sent_tokenized'] = model_data_preprocessing[\n            'sentence'].progress_apply(tokenize_sentence)\n\n        model_data_preprocessing[\n            'token_tag'] = model_data_preprocessing.progress_apply(\n                lambda x: get_tags(sent=x['sent_tokenized'], ep=x['dataset_label'])\n                if (isinstance(x['sent_tokenized'], list)) else np.nan, axis=1)\n\n        model_data_preprocessing['vocab_sent_tokenized'], model_data_preprocessing[\n            'sent_input_ids'], model_data_preprocessing[\n                'vocab_token_tag'], model_data_preprocessing['token_tag_ids'] = zip(\n                    *model_data_preprocessing.\n                    progress_apply(lambda x: vocab_sent_tokenize_label(\n                        sent_tokenized=x['sent_tokenized'], token_tag=x['token_tag'])\n                                   if isinstance(x['token_tag'], list) else np.nan,\n                                   axis=1))\n        return model_data_preprocessing\n    except Exception as e:\n        print(f\"Error in line no: {sys.exc_info()[2].tb_lineno}\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.872238Z","iopub.execute_input":"2021-06-13T13:23:33.872824Z","iopub.status.idle":"2021-06-13T13:23:33.882403Z","shell.execute_reply.started":"2021-06-13T13:23:33.872785Z","shell.execute_reply":"2021-06-13T13:23:33.881490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_data(input_ids, token_ids):\n    try:\n        max_token_length = 512\n        attention_mask = []\n        for input_ in tqdm(input_ids):\n            attention_mask.append(torch.ones(len(input_[:max_token_length])))\n\n        padded_attention_mask = torch.nn.utils.rnn.pad_sequence(attention_mask,\n                                                                batch_first=True,\n                                                                padding_value=0.0)\n\n        padded_input_ids = torch.nn.utils.rnn.pad_sequence(\n            [torch.tensor(input_[:max_token_length]) for input_ in input_ids],\n            batch_first=True,\n            padding_value=0.0)\n\n        padded_tags = torch.nn.utils.rnn.pad_sequence(\n            [torch.tensor(tag_[:max_token_length]) for tag_ in token_ids],\n            batch_first=True,\n            padding_value=0.0)\n        return padded_input_ids, padded_attention_mask, padded_tags\n    except Exception as e:\n        print(f\"Error in line no: {sys.exc_info()[2].tb_lineno}\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.883664Z","iopub.execute_input":"2021-06-13T13:23:33.884055Z","iopub.status.idle":"2021-06-13T13:23:33.894352Z","shell.execute_reply.started":"2021-06-13T13:23:33.884003Z","shell.execute_reply":"2021-06-13T13:23:33.893538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_dataloader(token_ids, masks, tags, batch_size=16, val=False):\n    try:\n        # wrap tensors\n        data = TensorDataset(token_ids, masks, tags)\n\n        if val:\n            # sampler for sampling the data during training\n            sampler = SequentialSampler(data)\n            \n            # dataLoader for validation set\n            dataloader = DataLoader(data,\n                                    sampler=sampler,\n                                    batch_size=batch_size)\n        else:    \n            # sampler for sampling the data during training\n            sampler = RandomSampler(data)\n            # dataLoader for train set\n            dataloader = DataLoader(data,\n                                    sampler=sampler,\n                                    batch_size=batch_size)\n        return dataloader\n    except Exception as e:\n        print(f\"Error in line no: {sys.exc_info()[2].tb_lineno}\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:23:33.895724Z","iopub.execute_input":"2021-06-13T13:23:33.896150Z","iopub.status.idle":"2021-06-13T13:23:33.905992Z","shell.execute_reply.started":"2021-06-13T13:23:33.896113Z","shell.execute_reply":"2021-06-13T13:23:33.905086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\ntokenizer = AutoTokenizer.from_pretrained('../input/coleridge-model-data')\ntag2idx = {\n    'O': 0,\n    'B': 1,\n    'I': 2,\n}\n\nidx2tag = {\n    \"0\": \"O\",\n    \"1\": \"B\",\n    \"2\": \"I\"\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:00.483073Z","iopub.execute_input":"2021-06-13T13:26:00.483425Z","iopub.status.idle":"2021-06-13T13:26:00.576999Z","shell.execute_reply.started":"2021-06-13T13:26:00.483393Z","shell.execute_reply":"2021-06-13T13:26:00.576171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data_v1 = train_data.groupby(['Id']).agg({'dataset_label':'|'.join}).reset_index()\ntraining_data_v1['dataset_label'] = training_data_v1['dataset_label'].apply(lambda x: x.split('|'))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:06.526658Z","iopub.execute_input":"2021-06-13T13:26:06.526974Z","iopub.status.idle":"2021-06-13T13:26:06.675769Z","shell.execute_reply.started":"2021-06-13T13:26:06.526945Z","shell.execute_reply":"2021-06-13T13:26:06.674916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth',500)\ntraining_data_v1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:06.677367Z","iopub.execute_input":"2021-06-13T13:26:06.677751Z","iopub.status.idle":"2021-06-13T13:26:06.689249Z","shell.execute_reply.started":"2021-06-13T13:26:06.677712Z","shell.execute_reply":"2021-06-13T13:26:06.688253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data_v1['sentence'] = np.NaN\nfor i in tqdm(range(0, len(training_data_v1))):\n    text = training_data_v1['Id'].iloc[i]\n    file_path = '../input/coleridgeinitiative-show-us-the-data/train//' + text + '.json'\n    file_id = file_path.split('/')[-1].split('.json')[0]\n    with open(file_path) as json_file:\n        data = json.load(json_file)\n\n    training_data_v1['sentence'].iloc[i] = ' '.join([x['text'].strip().lower() for x in data])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:06.691633Z","iopub.execute_input":"2021-06-13T13:26:06.692350Z","iopub.status.idle":"2021-06-13T13:26:25.078549Z","shell.execute_reply.started":"2021-06-13T13:26:06.692308Z","shell.execute_reply":"2021-06-13T13:26:25.077685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth',500)\ntraining_data_v1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:25.080261Z","iopub.execute_input":"2021-06-13T13:26:25.080616Z","iopub.status.idle":"2021-06-13T13:26:25.093334Z","shell.execute_reply.started":"2021-06-13T13:26:25.080576Z","shell.execute_reply":"2021-06-13T13:26:25.092347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Max token length for BERT is 512. Hence, we need to break down the sentences into smaller groups\n\ncreate_train_data = False # Make this true to create data\n\nif create_train_data:\n    training_data_for_model = pd.DataFrame(data = None, columns = ['Id','sentence'])\n    count = 0\n    for j in tqdm(range(0, len(training_data_v1))):\n        text_from_doc = tokenize_sentence(training_data_v1['sentence'].iloc[j])\n        parts = [' '.join(text_from_doc[i:i+512]) for i in range(0, len(text_from_doc), 500)]\n        for k in parts:\n            training_data_for_model.loc[count] = training_data_v1['Id'].iloc[j], k\n            count = count + 1\n            \n    pd.set_option('display.max_colwidth',500)\n    print(training_data_for_model.shape)\n    training_data_for_model.head()\n\n    print(training_data_for_model.shape)\n    training_data_for_model = training_data_for_model.merge(training_data_v1[['Id','dataset_label']], how = 'left', on = 'Id')\n    print(training_data_for_model.shape)\n\n    pd.set_option('display.max_colwidth',500)\n    print(training_data_for_model.shape)\n    training_data_for_model.head()\n\n    training_data_for_model['flag'] = training_data_for_model[['dataset_label','sentence']].apply(lambda x: ([1 if k.strip().lower() in x['sentence'].lower().strip() else 0 for k in x['dataset_label']]), axis = 1)\n\n    training_data_for_model['flag_sum'] = training_data_for_model['flag'].apply(lambda x: sum(x)) \n    training_data_for_model['flag_sum'] = np.where(training_data_for_model['flag_sum']>0,1,0)\n\n    training_data_for_model[training_data_for_model['Id']=='000efc17-13d8-433d-8f62-a3932fe4f3b8']\n\n    print('Sentences without labels:', training_data_for_model[training_data_for_model['flag_sum']==0].shape[0]/training_data_for_model.shape[0] * 100,'%')\n    print('Sentences with labels:', training_data_for_model[training_data_for_model['flag_sum']==1].shape[0]/training_data_for_model.shape[0] * 100,'%')\n\n    training_data_for_model_v1 = training_data_for_model[training_data_for_model['flag_sum']==1].reset_index(drop = True)\n\n    print(training_data_for_model_v1.shape)\n    training_data_for_model_v1.head()\n    \n    with open('training_data_for_model_v1.pkl', 'wb') as file:\n        pickle.dump(training_data_for_model_v1, file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:25.094840Z","iopub.execute_input":"2021-06-13T13:26:25.095379Z","iopub.status.idle":"2021-06-13T13:26:25.111432Z","shell.execute_reply.started":"2021-06-13T13:26:25.095340Z","shell.execute_reply":"2021-06-13T13:26:25.110475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/coleridgetrainingdata/training_data_for_model_v1.pkl', 'rb') as file:\n    training_data_for_model_v1 = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:25.112911Z","iopub.execute_input":"2021-06-13T13:26:25.113340Z","iopub.status.idle":"2021-06-13T13:26:26.230473Z","shell.execute_reply.started":"2021-06-13T13:26:25.113303Z","shell.execute_reply":"2021-06-13T13:26:26.229565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For train\nbatch_size=10\n\ncreate_train_data = False # Put true for training\n\nif create_train_data:\n    processed_sentence_tag_full_data = sent_tag_tokenization(data=training_data_for_model_v1)\n    processed_sentence_tag = processed_sentence_tag_full_data[['sent_input_ids', 'token_tag_ids']]\n\n    input_ids = processed_sentence_tag['sent_input_ids'].tolist()\n    token_ids = processed_sentence_tag['token_tag_ids'].tolist()\n\n    padded_input_ids, padded_attention_mask, padded_tags = pad_data(\n        input_ids=input_ids, \n        token_ids=token_ids\n    )\n\n    train_dataloader = create_dataloader(\n        token_ids=padded_input_ids, \n        masks=padded_attention_mask, \n        tags=padded_tags, \n        batch_size=batch_size, \n        val=False\n    )\n\n    with open('train_dataloader.pkl', 'wb') as file:\n        pickle.dump(train_dataloader, file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:26.231938Z","iopub.execute_input":"2021-06-13T13:26:26.232305Z","iopub.status.idle":"2021-06-13T13:26:26.240084Z","shell.execute_reply.started":"2021-06-13T13:26:26.232268Z","shell.execute_reply":"2021-06-13T13:26:26.239074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/coleridgetrainingdata/train_dataloader.pkl', 'rb') as file:\n    train_dataloader = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:26.243748Z","iopub.execute_input":"2021-06-13T13:26:26.244294Z","iopub.status.idle":"2021-06-13T13:26:28.453481Z","shell.execute_reply.started":"2021-06-13T13:26:26.244258Z","shell.execute_reply":"2021-06-13T13:26:28.452535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del train_data, train_data_cl_label_len_summary, train_data_cl_label_summary, training_data_for_model, training_data_for_model_v1, training_data_v1\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:28.455500Z","iopub.execute_input":"2021-06-13T13:26:28.455853Z","iopub.status.idle":"2021-06-13T13:26:28.703288Z","shell.execute_reply.started":"2021-06-13T13:26:28.455818Z","shell.execute_reply":"2021-06-13T13:26:28.700892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# config = AutoConfig.from_pretrained(\n#     'bert-base-uncased', \n#     num_labels=len(tag2idx),\n#     id2label=idx2tag,\n#     label2id=tag2idx\n# )\n# model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased', config=config)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:55.793853Z","iopub.execute_input":"2021-06-13T13:26:55.794210Z","iopub.status.idle":"2021-06-13T13:26:55.798194Z","shell.execute_reply.started":"2021-06-13T13:26:55.794178Z","shell.execute_reply":"2021-06-13T13:26:55.797088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nprint(device)\n\n# model = model.to(device)\n\n# optimizer = AdamW(model.parameters(),\n#                   lr = 1e-5) # learning rate\n\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:10.796774Z","iopub.execute_input":"2021-06-13T13:27:10.797123Z","iopub.status.idle":"2021-06-13T13:27:10.846549Z","shell.execute_reply.started":"2021-06-13T13:27:10.797091Z","shell.execute_reply":"2021-06-13T13:27:10.844552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_train(model, optimizer, loss_criteria, train_dataloader):\n    try:\n        model.train()\n\n        total_loss = 0\n        total_logits = []\n\n        # iterate over batches\n        for step, batch in enumerate(train_dataloader):\n\n            # progress update after every 50 batches.\n            if step % 50 == 0 and not step == 0:\n                print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n\n            # push the batch to gpu\n            batch = [r.to(device) for r in batch]\n\n            sent_id, mask, labels = batch\n            \n            gc.collect()\n            torch.cuda.empty_cache()\n            # clear previously calculated gradients\n            model.zero_grad()\n\n            # get model predictions for the current batch\n            logits = model(sent_id.to(device), mask.to(device))\n\n            # compute the loss between actual and predicted values\n            loss = loss_criteria(logits.logits.permute(0, 2, 1), labels)\n\n            # add on to the total loss\n            total_loss = total_loss + loss.item()\n\n            # backward pass to calculate the gradients\n            loss.backward()\n\n            # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n            # update parameters\n            optimizer.step()\n\n            # model predictions are stored on GPU. So, push it to CPU\n            logits = logits.logits.detach().cpu().numpy()\n\n            # append the model predictions\n            total_logits.append(logits)\n\n        # compute the training loss of the epoch\n        avg_loss = total_loss / len(train_dataloader)\n\n        total_logits = np.concatenate(total_logits, axis=0)\n\n\n        return avg_loss, total_logits\n    except Exception as e:\n        print(f\"Error during training the model on line: {sys.exc_info()[2].tb_lineno}\")\n        print(e)\n\n\n# function for evaluating the model\ndef do_evaluate(model, val_dataloader, loss_criteria):\n    print(\"\\nEvaluating...\")\n    \n    # deactivate dropout layers\n    model.eval()\n\n    total_loss, total_accuracy = 0, 0\n\n    # empty list to save the model predictions\n    total_logits = []\n\n    # iterate over batches\n    for step, batch in enumerate(val_dataloader):\n\n        # Progress update every 50 batches.\n        if step % 50 == 0 and not step == 0:\n            # Calculate elapsed time in minutes.\n            #             elapsed = format_time(time.time() - t0)\n\n            # Report progress.\n            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n\n        # push the batch to gpu\n        batch = [t.to(device) for t in batch]\n\n        sent_id, mask, labels = batch\n\n        # deactivate autograd\n        with torch.no_grad():\n\n            # model predictions\n            logits = model(sent_id.to(device), mask.to(device))\n\n            # compute the validation loss between actual and predicted values\n            loss = loss_criteria(logits.logits.permute(0, 2, 1), labels)\n\n            total_loss = total_loss + loss.item()\n\n            logits = logits.logits.detach().cpu().numpy()\n\n            total_logits.append(logits)\n\n    # compute the validation loss of the epoch\n    avg_loss = total_loss / len(val_dataloader)\n\n    # reshape the predictions in form of (number of samples, no. of classes)\n    total_logits = np.concatenate(total_logits, axis=0)\n\n    return avg_loss, total_logits","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:14.411102Z","iopub.execute_input":"2021-06-13T13:27:14.411427Z","iopub.status.idle":"2021-06-13T13:27:14.425198Z","shell.execute_reply.started":"2021-06-13T13:27:14.411399Z","shell.execute_reply":"2021-06-13T13:27:14.424063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_flag = False\n\nif train_flag:\n    %%time\n    epochs=1 ## Need to increase this and see better performance\n\n    # empty lists to store training and validation loss of each epoch\n    train_losses=[]\n\n    #for each epoch\n\n    for epoch in range(epochs):\n        start = time.time()\n        print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n\n        #train model\n        train_loss, _ = do_train(\n            model = model,\n            optimizer = optimizer, \n            loss_criteria = criterion, \n            train_dataloader = train_dataloader \n\n        )\n\n        model.save_pretrained('model_file')\n        tokenizer.save_pretrained('tokenizer_file')\n\n        # append training and validation loss\n        train_losses.append(train_loss)\n\n        print(f\"Time taken: {time.time() - start}\")\n        print(f'\\nTraining Loss: {train_loss:.3f}')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:17.371674Z","iopub.execute_input":"2021-06-13T13:27:17.371993Z","iopub.status.idle":"2021-06-13T13:27:17.379696Z","shell.execute_reply.started":"2021-06-13T13:27:17.371963Z","shell.execute_reply":"2021-06-13T13:27:17.378745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load fine-tuned model\nmodel = AutoModelForTokenClassification.from_pretrained('../input/coleridge-model-data')\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:19.613964Z","iopub.execute_input":"2021-06-13T13:27:19.614422Z","iopub.status.idle":"2021-06-13T13:27:33.991791Z","shell.execute_reply.started":"2021-06-13T13:27:19.614388Z","shell.execute_reply":"2021-06-13T13:27:33.990831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction_from_logits(logits):\n    try:\n        tag_prob = nn.Softmax(dim=2)(logits)\n        tag_prediction = torch.argmax(tag_prob, dim=2).detach().cpu().numpy()\n        return tag_prediction\n    except Exception as e:\n        print(f\"Error in line: {sys.exc_info()[2].tb_lineno}\")\n        print(e)\n        \ndef classification_result(tag2idx, c_tag_id):\n    try:\n        prediction_result = []\n        for sent_ in c_tag_id:\n            prediction_result.append(\n                list(map(lambda x: list(tag2idx.keys())[list(tag2idx.values()).index(x)], sent_))\n            )\n            \n        tagged_entity = np.concatenate(prediction_result, axis=0)\n        return tagged_entity\n    except Exception as e:\n        print(f\"Error in line: {sys.exc_info()[2].tb_lineno}\")\n        print(e) ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:37.907577Z","iopub.execute_input":"2021-06-13T13:27:37.907951Z","iopub.status.idle":"2021-06-13T13:27:37.915082Z","shell.execute_reply.started":"2021-06-13T13:27:37.907922Z","shell.execute_reply":"2021-06-13T13:27:37.914156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_file","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:37.947528Z","iopub.execute_input":"2021-06-13T13:27:37.947774Z","iopub.status.idle":"2021-06-13T13:27:37.959372Z","shell.execute_reply.started":"2021-06-13T13:27:37.947750Z","shell.execute_reply":"2021-06-13T13:27:37.958124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For test data\ntest_data_v1 = submission_file[['Id']]\ntest_data_v1['sentence'] = np.NaN\nfor i in tqdm(range(0, len(submission_file))):\n    text = submission_file['Id'].iloc[i]\n    file_path = '../input/coleridgeinitiative-show-us-the-data/test//' + text + '.json'\n    file_id = file_path.split('/')[-1].split('.json')[0]\n    with open(file_path) as json_file:\n        data = json.load(json_file)\n\n    test_data_v1['sentence'].iloc[i] = ' '.join([x['text'].strip().lower() for x in data])\n    \n# Max token length for BERT is 512. Hence, we need to break down the sentences into smaller groups\ntest_data_for_model = pd.DataFrame(data = None, columns = ['Id','sentence'])\ncount = 0\nfor j in tqdm(range(0, len(test_data_v1))):\n    text_from_doc = tokenize_sentence(test_data_v1['sentence'].iloc[j])\n    parts = [' '.join(text_from_doc[i:i+512]) for i in range(0, len(text_from_doc), 500)]\n    for k in parts:\n        test_data_for_model.loc[count] = test_data_v1['Id'].iloc[j], k\n        count = count + 1","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:37.960694Z","iopub.execute_input":"2021-06-13T13:27:37.961001Z","iopub.status.idle":"2021-06-13T13:27:38.255964Z","shell.execute_reply.started":"2021-06-13T13:27:37.960973Z","shell.execute_reply":"2021-06-13T13:27:38.254749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_for_model['dataset_label'] = [[''] for x in range(0,len(test_data_for_model))]\nprint(test_data_for_model.shape)\ntest_data_for_model.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:38.257828Z","iopub.execute_input":"2021-06-13T13:27:38.258200Z","iopub.status.idle":"2021-06-13T13:27:38.272208Z","shell.execute_reply.started":"2021-06-13T13:27:38.258161Z","shell.execute_reply":"2021-06-13T13:27:38.270994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For test data - tokenization + attention masks\nprocessed_sentence_tag_full_data = sent_tag_tokenization(data=test_data_for_model)\nprocessed_sentence_tag = processed_sentence_tag_full_data[['sent_input_ids', 'token_tag_ids']]\n\ninput_ids = processed_sentence_tag['sent_input_ids'].tolist()\ntoken_ids = processed_sentence_tag['token_tag_ids'].tolist()\n\ntest_padded_input_ids, test_padded_attention_mask, test_padded_tags = pad_data(\n    input_ids=input_ids, \n    token_ids=token_ids\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:38.274258Z","iopub.execute_input":"2021-06-13T13:27:38.274646Z","iopub.status.idle":"2021-06-13T13:27:41.539549Z","shell.execute_reply.started":"2021-06-13T13:27:38.274608Z","shell.execute_reply":"2021-06-13T13:27:41.538638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get predictions for test data\nfor i in range(0,10):\n    with torch.no_grad():\n        logits = model(test_padded_input_ids[i:i+1].to(device), test_padded_attention_mask[i:i+1].to(device))\n        preds = get_prediction_from_logits(logits=logits['logits'])\n    print(sum(preds[0]))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:41.543207Z","iopub.execute_input":"2021-06-13T13:27:41.545139Z","iopub.status.idle":"2021-06-13T13:27:42.551769Z","shell.execute_reply.started":"2021-06-13T13:27:41.545101Z","shell.execute_reply":"2021-06-13T13:27:42.551023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predicted_labels(sent_padded_input_ids, sent_padded_attention_mask):\n    try:\n        with torch.no_grad():\n            logits = model(sent_padded_input_ids.to(device), sent_padded_attention_mask.to(device))\n            c_tag_id = get_prediction_from_logits(logits=logits['logits'])        \n        test_ids = np.squeeze(sent_padded_input_ids.reshape(1, -1)).detach().cpu().numpy()\n        preds = classification_result(\n            tag2idx = tag2idx, \n            c_tag_id = c_tag_id\n        )        \n        test_tokens = [tokenizer.convert_ids_to_tokens(int(x)) for x in test_ids]\n        \n        final_out_ls = []\n        #final_out_ls1 = []\n        temp = []\n        for _idx, _tag in enumerate(preds):\n            if _tag in ['B', 'I']:\n                temp.append(test_tokens[_idx])\n            else:\n                if len(temp)>0:\n                    e = ''\n                    for t2 in temp:\n                        if t2.startswith('##'): e = e+t2.lstrip('##')\n                        else: e = e + ' ' +t2\n                    # final_out_ls1.append(temp)\n                    final_out_ls.append(e.strip())\n                temp = []        \n        \n        return final_out_ls\n    except Exception as e:\n        print(f\"Error in line: {sys.exc_info()[2].tb_lineno}\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:42.552966Z","iopub.execute_input":"2021-06-13T13:27:42.553324Z","iopub.status.idle":"2021-06-13T13:27:42.566827Z","shell.execute_reply.started":"2021-06-13T13:27:42.553287Z","shell.execute_reply":"2021-06-13T13:27:42.564482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_labels = []\nfor i in tqdm(range(test_data_for_model.shape[0])):\n    predicted_labels.append(get_predicted_labels(test_padded_input_ids[i:i+1].to(device), test_padded_attention_mask[i:i+1].to(device)))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:42.568063Z","iopub.execute_input":"2021-06-13T13:27:42.568439Z","iopub.status.idle":"2021-06-13T13:27:45.121479Z","shell.execute_reply.started":"2021-06-13T13:27:42.568389Z","shell.execute_reply":"2021-06-13T13:27:45.120124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.122753Z","iopub.execute_input":"2021-06-13T13:27:45.123112Z","iopub.status.idle":"2021-06-13T13:27:45.130543Z","shell.execute_reply.started":"2021-06-13T13:27:45.123077Z","shell.execute_reply":"2021-06-13T13:27:45.129486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_for_model['predicted_label'] = predicted_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.134339Z","iopub.execute_input":"2021-06-13T13:27:45.134746Z","iopub.status.idle":"2021-06-13T13:27:45.140591Z","shell.execute_reply.started":"2021-06-13T13:27:45.134708Z","shell.execute_reply":"2021-06-13T13:27:45.139574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_for_model['PredictionString'] = test_data_for_model['predicted_label'].apply(lambda x: ' '.join(x))\ntest_data_for_model['PredictionString_clean'] = test_data_for_model['PredictionString'].apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.142791Z","iopub.execute_input":"2021-06-13T13:27:45.143263Z","iopub.status.idle":"2021-06-13T13:27:45.152283Z","shell.execute_reply.started":"2021-06-13T13:27:45.143158Z","shell.execute_reply":"2021-06-13T13:27:45.151444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results_ner = test_data_for_model[test_data_for_model['PredictionString_clean']!=''].groupby('Id').agg({'PredictionString_clean':' | '.join}).reset_index().rename(columns = {'PredictionString_clean':'PredictionString'})","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.153691Z","iopub.execute_input":"2021-06-13T13:27:45.154068Z","iopub.status.idle":"2021-06-13T13:27:45.164370Z","shell.execute_reply.started":"2021-06-13T13:27:45.154030Z","shell.execute_reply":"2021-06-13T13:27:45.163355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results_ner","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.165736Z","iopub.execute_input":"2021-06-13T13:27:45.166122Z","iopub.status.idle":"2021-06-13T13:27:45.184215Z","shell.execute_reply.started":"2021-06-13T13:27:45.166087Z","shell.execute_reply":"2021-06-13T13:27:45.183121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lookup approach","metadata":{}},{"cell_type":"code","source":"test_data_for_model['Predicted_labels - Lookup approach'] = test_data_for_model['sentence'].apply(lambda x: keywordprocessor.extract_keywords(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.185523Z","iopub.execute_input":"2021-06-13T13:27:45.185911Z","iopub.status.idle":"2021-06-13T13:27:45.302534Z","shell.execute_reply.started":"2021-06-13T13:27:45.185875Z","shell.execute_reply":"2021-06-13T13:27:45.301690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_for_model[test_data_for_model['Predicted_labels - Lookup approach'].str.len() > 0].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.303780Z","iopub.execute_input":"2021-06-13T13:27:45.304129Z","iopub.status.idle":"2021-06-13T13:27:45.336983Z","shell.execute_reply.started":"2021-06-13T13:27:45.304094Z","shell.execute_reply":"2021-06-13T13:27:45.336183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_for_model['Predicted_labels - Lookup approach'] = test_data_for_model['Predicted_labels - Lookup approach'].apply(lambda x: ' | '.join(x))\ntest_data_for_model['Predicted_labels - Lookup approach'] = test_data_for_model['Predicted_labels - Lookup approach'].apply(lambda x: clean_text(x))\nfinal_test_results_lookup = test_data_for_model[test_data_for_model['Predicted_labels - Lookup approach'].str.len() > 0].groupby('Id').agg({'Predicted_labels - Lookup approach':' | '.join}).reset_index().rename(columns = {'Predicted_labels - Lookup approach':'PredictionString'})","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.338284Z","iopub.execute_input":"2021-06-13T13:27:45.338613Z","iopub.status.idle":"2021-06-13T13:27:45.351900Z","shell.execute_reply.started":"2021-06-13T13:27:45.338579Z","shell.execute_reply":"2021-06-13T13:27:45.350933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results_lookup","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.353409Z","iopub.execute_input":"2021-06-13T13:27:45.353907Z","iopub.status.idle":"2021-06-13T13:27:45.367175Z","shell.execute_reply.started":"2021-06-13T13:27:45.353872Z","shell.execute_reply":"2021-06-13T13:27:45.366296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results = submission_file[['Id']].merge(final_test_results_lookup, how = 'left', on = 'Id')\nfinal_test_results = final_test_results.merge(final_test_results_ner, how = 'left', on = 'Id')\nfinal_test_results['PredictionString'] = final_test_results['PredictionString_x'].fillna('') + ' | ' + final_test_results['PredictionString_y'].fillna('')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.368543Z","iopub.execute_input":"2021-06-13T13:27:45.368966Z","iopub.status.idle":"2021-06-13T13:27:45.384377Z","shell.execute_reply.started":"2021-06-13T13:27:45.368917Z","shell.execute_reply":"2021-06-13T13:27:45.383538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.387295Z","iopub.execute_input":"2021-06-13T13:27:45.387596Z","iopub.status.idle":"2021-06-13T13:27:45.400012Z","shell.execute_reply.started":"2021-06-13T13:27:45.387559Z","shell.execute_reply":"2021-06-13T13:27:45.398801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"manual_noise_list_identified = ['international of']","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.401690Z","iopub.execute_input":"2021-06-13T13:27:45.402324Z","iopub.status.idle":"2021-06-13T13:27:45.406856Z","shell.execute_reply.started":"2021-06-13T13:27:45.402286Z","shell.execute_reply":"2021-06-13T13:27:45.405618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_noise(text):\n    tokens = text.split(' | ')\n    tokens = list(set(tokens))\n    tokens = [x for x in tokens if (len(x)>=5) & (x not in manual_noise_list_identified)]\n    result = ' | '.join(tokens)\n    return result","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.408473Z","iopub.execute_input":"2021-06-13T13:27:45.409391Z","iopub.status.idle":"2021-06-13T13:27:45.416069Z","shell.execute_reply.started":"2021-06-13T13:27:45.409346Z","shell.execute_reply":"2021-06-13T13:27:45.415273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove noise\nfinal_test_results['PredictionString'] = final_test_results['PredictionString'].apply(lambda x: remove_noise(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.418658Z","iopub.execute_input":"2021-06-13T13:27:45.419072Z","iopub.status.idle":"2021-06-13T13:27:45.426099Z","shell.execute_reply.started":"2021-06-13T13:27:45.419008Z","shell.execute_reply":"2021-06-13T13:27:45.424925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results = submission_file[['Id']].merge(final_test_results, on = 'Id', how = 'left')[['Id','PredictionString']]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.427589Z","iopub.execute_input":"2021-06-13T13:27:45.428052Z","iopub.status.idle":"2021-06-13T13:27:45.439242Z","shell.execute_reply.started":"2021-06-13T13:27:45.427991Z","shell.execute_reply":"2021-06-13T13:27:45.438388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.440683Z","iopub.execute_input":"2021-06-13T13:27:45.441096Z","iopub.status.idle":"2021-06-13T13:27:45.453174Z","shell.execute_reply.started":"2021-06-13T13:27:45.441055Z","shell.execute_reply":"2021-06-13T13:27:45.451872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_results.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:45.454743Z","iopub.execute_input":"2021-06-13T13:27:45.455147Z","iopub.status.idle":"2021-06-13T13:27:45.467856Z","shell.execute_reply.started":"2021-06-13T13:27:45.455110Z","shell.execute_reply":"2021-06-13T13:27:45.466800Z"},"trusted":true},"execution_count":null,"outputs":[]}]}