{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-22T07:06:09.248031Z","iopub.execute_input":"2021-05-22T07:06:09.248419Z","iopub.status.idle":"2021-05-22T07:06:09.252432Z","shell.execute_reply.started":"2021-05-22T07:06:09.248339Z","shell.execute_reply":"2021-05-22T07:06:09.251486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This notebook is the continuation from my Notebook on [Bert for Token Classification - Training](http://https://www.kaggle.com/thanish/bert-for-token-classification-training). \nPlease do check it out for the training code","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport json\nimport re\nfrom nltk.tokenize import sent_tokenize \nfrom transformers import BertTokenizer, AutoTokenizer\nfrom torch.utils.data import DataLoader\nimport torch\nimport torch.nn as nn\nimport transformers\nfrom tqdm import tqdm\nimport glob\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:21.303495Z","iopub.execute_input":"2021-05-24T08:12:21.303920Z","iopub.status.idle":"2021-05-24T08:12:24.408514Z","shell.execute_reply.started":"2021-05-24T08:12:21.303834Z","shell.execute_reply":"2021-05-24T08:12:24.407675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config ","metadata":{}},{"cell_type":"code","source":"platform = 'Kaggle'\nmodel_name = 'epoch_14_model_sage_bert_base_uncased.bin'\n\nif platform == 'Kaggle':\n    bert_path = '../input/huggingface-bert/bert-base-uncased'\n    train_path = '/kaggle/input/coleridgeinitiative-show-us-the-data/train/*'\n    test_path = '/kaggle/input/coleridgeinitiative-show-us-the-data/test/*'\n    model_path = '../input/d/thanish/coleridgemodels/'\n\nconfig = {'MAX_LEN': 128,\n          'tokenizer': AutoTokenizer.from_pretrained(bert_path , do_lower_case=True),\n          'batch_size': 20,\n          'Epoch': 10,\n          'train_path': train_path, \n          'test_path': test_path, \n          'device': 'cuda' if torch.cuda.is_available() else 'cpu',\n          'bert_path': bert_path,\n          'model_path': model_path,\n          'model_name': model_name\n         }","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.410800Z","iopub.execute_input":"2021-05-24T08:12:24.411049Z","iopub.status.idle":"2021-05-24T08:12:24.576706Z","shell.execute_reply.started":"2021-05-24T08:12:24.411024Z","shell.execute_reply":"2021-05-24T08:12:24.575881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.580110Z","iopub.execute_input":"2021-05-24T08:12:24.580358Z","iopub.status.idle":"2021-05-24T08:12:24.586469Z","shell.execute_reply.started":"2021-05-24T08:12:24.580334Z","shell.execute_reply":"2021-05-24T08:12:24.585618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_joining(data_dict_id):\n#     '''\n#     This function is to join all the text data from different sections in the json to a single\n#     text file. \n#     '''\n#     data_length = len(data_dict_id)\n\n#     temp = [data_dict_id[i]['text'] for i in range(data_length)]\n# #     temp = [data_dict_id[i]['text'] for i in range(0, 1)]\n#     temp = '. '.join(temp)\n    \n#     return temp\n\n\ndef data_joining(data_dict_id):\n    '''\n    This function is to join all the text data from different sections in the json to a single\n    text file. \n    '''\n\n    sent_list = []\n    for i in range(len(data_dict_id)):\n        text = data_dict_id[i]['text']\n        text = clean_text(text).strip()\n        text = re.sub(' +', ' ', text)\n            \n#         if len(text.split(\" \"))>15: #If the text is greater than 10 words.\n#             temp = [text if any(word in text.lower() for word in ['data', 'study']) else '']\n#             sent_list.append(temp[-1])\n\n        if len(text.split(\" \"))>15: #If the text is greater than 20 words.\n            sent_list.append(text)\n            \n    sent_list = list(set(sent_list))\n    final_sentence = '. '.join(sent_list)\n            \n    return final_sentence","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.588095Z","iopub.execute_input":"2021-05-24T08:12:24.588542Z","iopub.status.idle":"2021-05-24T08:12:24.596033Z","shell.execute_reply.started":"2021-05-24T08:12:24.588504Z","shell.execute_reply":"2021-05-24T08:12:24.595003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# for id in test_data_dict.keys():\n#     print(id)\n#     print(len(data_joining(test_data_dict[id]).split(\" \")))\n#     print(len(data_joining_2(test_data_dict[id]).split(\" \")))\n    \n    \n# # 8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60\n# # 7540\n# # 7689\n# # 2100032a-7c33-4bff-97ef-690822c43466\n# # 3652\n# # 3128\n# # 2f392438-e215-4169-bebf-21ac4ff253e1\n# # 28673\n# # 23368\n# # 3f316b38-1a24-45a9-8d8c-4e05a42257c6\n# # 10671\n# # 9868\n# # CPU times: user 67.9 ms, sys: 3.32 ms, total: 71.2 ms\n# # Wall time: 69.4 ms","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.597464Z","iopub.execute_input":"2021-05-24T08:12:24.597887Z","iopub.status.idle":"2021-05-24T08:12:24.610482Z","shell.execute_reply.started":"2021-05-24T08:12:24.597849Z","shell.execute_reply":"2021-05-24T08:12:24.609624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the test data","metadata":{}},{"cell_type":"code","source":"def read_test_json(test_data_folder):\n    '''\n    This function reads all the json input files and return a dictionary containing the id as the key\n    and all the contents of the json as values\n    '''\n\n    test_text_data = {}\n    total_files = len(glob.glob(test_data_folder))\n    \n    for i, test_json_loc in enumerate(glob.glob(test_data_folder)):\n        filename = test_json_loc.split(\"/\")[-1][:-5]\n\n        with open(test_json_loc, 'r') as f:\n            test_text_data[filename] = json.load(f)\n\n        if (i%1000) == 0:\n            print(f\"Completed {i}/{total_files}\")\n\n    print(\"All files read\")\n    return test_text_data","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.612177Z","iopub.execute_input":"2021-05-24T08:12:24.612549Z","iopub.status.idle":"2021-05-24T08:12:24.621740Z","shell.execute_reply.started":"2021-05-24T08:12:24.612513Z","shell.execute_reply":"2021-05-24T08:12:24.620946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_dict = read_test_json(test_data_folder=config['test_path'])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.623201Z","iopub.execute_input":"2021-05-24T08:12:24.623663Z","iopub.status.idle":"2021-05-24T08:12:24.658787Z","shell.execute_reply.started":"2021-05-24T08:12:24.623603Z","shell.execute_reply":"2021-05-24T08:12:24.657906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the saved model","metadata":{}},{"cell_type":"code","source":"# initializing the model\nmodel = transformers.BertForTokenClassification.from_pretrained(config['bert_path'],  num_labels = 3)\nmodel = nn.DataParallel(model)\n\n# Reading the trained checkpoint model\ntrained_model_name = config['model_path'] + config['model_name']\nprint(\"Trained model checkpoint:\", trained_model_name)\ncheckpoint = torch.load(trained_model_name, map_location = config['device'])\nprint(\"Checkpoint loaded\")\n\n# Matching the trained checkpoint model to the initialized model\nmodel.load_state_dict(checkpoint)\nprint(\"Model loaded with all keys matching with the checkpoint\")","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:24.660951Z","iopub.execute_input":"2021-05-24T08:12:24.661452Z","iopub.status.idle":"2021-05-24T08:12:41.688486Z","shell.execute_reply.started":"2021-05-24T08:12:24.661412Z","shell.execute_reply":"2021-05-24T08:12:41.687024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction function","metadata":{}},{"cell_type":"code","source":"# Prediction\ndef prediction_fn(tokenized_sub_sentence):\n\n    tkns = tokenized_sub_sentence\n    indexed_tokens = config['tokenizer'].convert_tokens_to_ids(tkns)\n    segments_ids = [0] * len(indexed_tokens)\n\n    tokens_tensor = torch.tensor([indexed_tokens]).to(config['device'])\n    segments_tensors = torch.tensor([segments_ids]).to(config['device'])\n    \n    model.eval()\n    with torch.no_grad():\n        logit = model(tokens_tensor, \n                      token_type_ids=None,\n                      attention_mask=segments_tensors)\n\n        logit_new = logit[0].argmax(2).detach().cpu().numpy().tolist()\n        prediction = logit_new[0]\n\n#         print(tkns)\n#         print(prediction)\n        \n        kword = ''\n        kword_list = []\n\n        for k, j in enumerate(prediction):\n            if (len(prediction)>1):\n\n                if (j!=0) & (k==0):\n                    #if it's the first word in the first position\n                    #print('At begin first word')\n                    begin = tkns[k]\n                    kword = begin\n\n                elif (j!=0) & (k>=1) & (prediction[k-1]==0):\n                    #begin word is in the middle of the sentence\n                    begin = tkns[k]\n                    previous = tkns[k-1]\n\n                    if begin.startswith('##'):\n                        kword = previous + begin[2:]\n                    else:\n                        kword = begin\n\n                    if k == (len(prediction) - 1):\n                        #print('begin and end word is the last word of the sentence')\n                        kword_list.append(kword.rstrip().lstrip())\n\n                elif (j!=0) & (k>=1) & (prediction[k-1]!=0):\n                    # intermediate word of the same keyword\n                    inter = tkns[k]\n\n                    if inter.startswith('##'):\n                        kword = kword + \"\" + inter[2:]\n                    else:\n                        kword = kword + \" \" + inter\n\n\n                    if k == (len(prediction) - 1):\n                        #print('begin and end')\n                        kword_list.append(kword.rstrip().lstrip())\n\n                elif (j==0) & (k>=1) & (prediction[k-1] !=0):\n                    # End of a keywords but not end of sentence.\n                    kword_list.append(kword.rstrip().lstrip())\n                    kword = ''\n                    inter = ''\n            else:\n                if (j!=0):\n                    begin = tkns[k]\n                    kword = begin\n                    kword_list.append(kword.rstrip().lstrip())\n#         print(kword_list)\n#         print(\"\")\n    return kword_list\n","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:41.690153Z","iopub.execute_input":"2021-05-24T08:12:41.690495Z","iopub.status.idle":"2021-05-24T08:12:41.704817Z","shell.execute_reply.started":"2021-05-24T08:12:41.690461Z","shell.execute_reply":"2021-05-24T08:12:41.703811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def long_sent_split(long_tokens):\n    '''\n    If the token length is >the max length this function splits it into mutiple list of specified smaller max_length\n    '''\n    \n    start = 0\n    end = len(long_tokens)\n    max_length = 64\n\n    final_long_tok_split = []\n    for i in range(start, end, max_length):\n        temp = long_tokens[i: (i + max_length)]\n        final_long_tok_split.append(temp)\n    return final_long_tok_split","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:41.706195Z","iopub.execute_input":"2021-05-24T08:12:41.706568Z","iopub.status.idle":"2021-05-24T08:12:41.717849Z","shell.execute_reply.started":"2021-05-24T08:12:41.706530Z","shell.execute_reply":"2021-05-24T08:12:41.716951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(data_dict):\n    \n    results = {}\n\n    for i, Id in enumerate(data_dict.keys()):\n        current_id_predictions = []\n        \n#         print(Id)\n        sentences = data_joining(data_dict[Id])\n        sentence_tokens = sent_tokenize(sentences)\n        \n        for sub_sentence in sentence_tokens:\n            cleaned_sub_sentence = clean_text(sub_sentence)\n        \n            # Tokenize the sentence\n            tokenized_sub_sentence = config['tokenizer'].tokenize(cleaned_sub_sentence)\n            \n            if len(tokenized_sub_sentence) == 0:\n                # If the tokenized sentence are empty\n                sub_sentence_prediction_kword_list = []\n                \n            elif len(tokenized_sub_sentence) <= 512:\n                # If the tokenized sentence are less than 512\n                sub_sentence_prediction_kword_list = prediction_fn(tokenized_sub_sentence)\n\n            else:\n                # If the tokenized sentence are >512 which is long sentences\n                long_sent_kword_list = []\n                \n                tokenized_sub_sentence_tok_split = long_sent_split(tokenized_sub_sentence)\n                for i, sent_tok in enumerate(tokenized_sub_sentence_tok_split):\n                    if len(sent_tok) != 0:\n                        kword_list = prediction_fn(sent_tok)\n                        long_sent_kword_list.append(kword_list)\n                flat_long_sent_kword = [item for sublist in long_sent_kword_list for item in sublist]\n                sub_sentence_prediction_kword_list = flat_long_sent_kword\n                            \n            if len(sub_sentence_prediction_kword_list) !=0:\n                current_id_predictions = current_id_predictions + sub_sentence_prediction_kword_list\n\n        results[Id] = list(set(current_id_predictions))\n                \n    print(\"All predictions completed\")\n    \n    return results","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:41.720737Z","iopub.execute_input":"2021-05-24T08:12:41.720982Z","iopub.status.idle":"2021-05-24T08:12:41.732164Z","shell.execute_reply.started":"2021-05-24T08:12:41.720958Z","shell.execute_reply":"2021-05-24T08:12:41.731353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults = get_predictions(data_dict = test_data_dict)\n# results","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:41.733209Z","iopub.execute_input":"2021-05-24T08:12:41.733705Z","iopub.status.idle":"2021-05-24T08:12:54.532739Z","shell.execute_reply.started":"2021-05-24T08:12:41.733672Z","shell.execute_reply":"2021-05-24T08:12:54.531855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_few_word_prediction(prediction_dict):\n    final_result = {}\n    for ID in prediction_dict.keys():\n        temp = []\n\n        for pred in prediction_dict[ID]:\n            pred_split = pred.split(\" \")\n#             print(ID, pred_split)\n            condition1 = len(pred_split)<=2\n            condition2 = 'adni' not in pred\n            condition3 = 'cccsl' not in pred\n            condition4 = 'ibtracs' not in pred\n            condition5 = 'slosh model' not in pred\n            \n            if condition1 & condition2 & condition3 & condition4 & condition5:\n                pass\n            else:\n                temp.append(pred)\n        final_result[ID] = temp\n        \n    return final_result\n\nresults = remove_few_word_prediction(prediction_dict=results)\nresults","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:12:54.534028Z","iopub.execute_input":"2021-05-24T08:12:54.534522Z","iopub.status.idle":"2021-05-24T08:12:54.546568Z","shell.execute_reply.started":"2021-05-24T08:12:54.534482Z","shell.execute_reply":"2021-05-24T08:12:54.545553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame({'Id': list(results.keys()),\n                       'PredictionString': list(results.values())})\nsub_df.PredictionString = sub_df.PredictionString.apply(lambda x : \"|\".join(x))\nsub_df","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:13:32.982664Z","iopub.execute_input":"2021-05-24T08:13:32.982991Z","iopub.status.idle":"2021-05-24T08:13:32.996005Z","shell.execute_reply.started":"2021-05-24T08:13:32.982962Z","shell.execute_reply":"2021-05-24T08:13:32.995184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:13:33.568951Z","iopub.execute_input":"2021-05-24T08:13:33.569811Z","iopub.status.idle":"2021-05-24T08:13:33.802941Z","shell.execute_reply.started":"2021-05-24T08:13:33.569708Z","shell.execute_reply":"2021-05-24T08:13:33.802069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ------------------------------------------- Consider upvoting if you like it :) ------------------------------------------- ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}