{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:27:40.676048Z","iopub.execute_input":"2021-06-04T08:27:40.676391Z","iopub.status.idle":"2021-06-04T08:27:40.680781Z","shell.execute_reply.started":"2021-06-04T08:27:40.676361Z","shell.execute_reply":"2021-06-04T08:27:40.679569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:27:40.685691Z","iopub.execute_input":"2021-06-04T08:27:40.686611Z","iopub.status.idle":"2021-06-04T08:29:04.246855Z","shell.execute_reply.started":"2021-06-04T08:27:40.686565Z","shell.execute_reply":"2021-06-04T08:29:04.245699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T08:29:04.252457Z","iopub.execute_input":"2021-06-04T08:29:04.254689Z","iopub.status.idle":"2021-06-04T08:29:04.263763Z","shell.execute_reply.started":"2021-06-04T08:29:04.254646Z","shell.execute_reply":"2021-06-04T08:29:04.262947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\n# Split train test data again to verify the accuracy using our own metrics.\ntrain, train_test = train_test_split(train,test_size=0.2, random_state = 1734)\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:04.268648Z","iopub.execute_input":"2021-06-04T08:29:04.271068Z","iopub.status.idle":"2021-06-04T08:29:41.603800Z","shell.execute_reply.started":"2021-06-04T08:29:04.271028Z","shell.execute_reply":"2021-06-04T08:29:41.603025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\n# sample_submission = pd.read_csv(sample_submission_path)\n\n# paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n# for paper_id in sample_submission['Id']:\n#     with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n#         paper = json.load(f)\n#         papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:41.606060Z","iopub.execute_input":"2021-06-04T08:29:41.606324Z","iopub.status.idle":"2021-06-04T08:29:41.613243Z","shell.execute_reply.started":"2021-06-04T08:29:41.606299Z","shell.execute_reply":"2021-06-04T08:29:41.612310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This section is for the testing on training data\nfor paper_id in train_test['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:41.617656Z","iopub.execute_input":"2021-06-04T08:29:41.617934Z","iopub.status.idle":"2021-06-04T08:29:43.673526Z","shell.execute_reply.started":"2021-06-04T08:29:41.617910Z","shell.execute_reply":"2021-06-04T08:29:43.672759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.676155Z","iopub.execute_input":"2021-06-04T08:29:43.676420Z","iopub.status.idle":"2021-06-04T08:29:43.722397Z","shell.execute_reply.started":"2021-06-04T08:29:43.676392Z","shell.execute_reply":"2021-06-04T08:29:43.721549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.723958Z","iopub.execute_input":"2021-06-04T08:29:43.724309Z","iopub.status.idle":"2021-06-04T08:29:43.730289Z","shell.execute_reply.started":"2021-06-04T08:29:43.724273Z","shell.execute_reply":"2021-06-04T08:29:43.729377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# literal_preds = []\n\n# for paper_id in sample_submission['Id']:\n#     paper = papers[paper_id]\n#     text_1 = '. '.join(section['text'] for section in paper).lower()\n#     text_2 = totally_clean_text(text_1)\n    \n#     labels = set()\n#     for label in all_labels:\n#         if label in text_1 or label in text_2:\n#             labels.add(clean_text(label))\n    \n#     literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.731582Z","iopub.execute_input":"2021-06-04T08:29:43.731985Z","iopub.status.idle":"2021-06-04T08:29:43.739837Z","shell.execute_reply.started":"2021-06-04T08:29:43.731950Z","shell.execute_reply":"2021-06-04T08:29:43.739058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # This section is for the testing on training data\n# literal_preds = []\n\n# for paper_id in train_test['Id'].unique():\n#     paper = papers[paper_id]\n#     text_1 = '. '.join(section['text'] for section in paper).lower()\n#     text_2 = totally_clean_text(text_1)\n    \n#     labels = set()\n#     for label in all_labels:\n#         if label in text_1 or label in text_2:\n#             labels.add(clean_text(label))\n    \n#     literal_preds.append('|'.join(labels))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.741362Z","iopub.execute_input":"2021-06-04T08:29:43.741755Z","iopub.status.idle":"2021-06-04T08:29:43.749035Z","shell.execute_reply.started":"2021-06-04T08:29:43.741692Z","shell.execute_reply":"2021-06-04T08:29:43.748438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.750326Z","iopub.execute_input":"2021-06-04T08:29:43.750868Z","iopub.status.idle":"2021-06-04T08:29:43.759050Z","shell.execute_reply.started":"2021-06-04T08:29:43.750832Z","shell.execute_reply":"2021-06-04T08:29:43.758343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/pytorch-bert-for-named-entity-recognition/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/pytorch-bert-for-named-entity-recognition/train_ner.json'\nVAL_PATH = '../input/pytorch-bert-for-named-entity-recognition/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.760644Z","iopub.execute_input":"2021-06-04T08:29:43.761028Z","iopub.status.idle":"2021-06-04T08:29:43.773741Z","shell.execute_reply.started":"2021-06-04T08:29:43.760992Z","shell.execute_reply":"2021-06-04T08:29:43.772843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:43.774996Z","iopub.execute_input":"2021-06-04T08:29:43.775339Z","iopub.status.idle":"2021-06-04T08:29:44.079095Z","shell.execute_reply.started":"2021-06-04T08:29:43.775305Z","shell.execute_reply":"2021-06-04T08:29:44.078197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:44.080361Z","iopub.execute_input":"2021-06-04T08:29:44.080877Z","iopub.status.idle":"2021-06-04T08:29:44.087523Z","shell.execute_reply.started":"2021-06-04T08:29:44.080838Z","shell.execute_reply":"2021-06-04T08:29:44.086747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_rows = [] # test data in NER format\n# paper_length = [] # store the number of sentences each paper has\n\n# for paper_id in sample_submission['Id']:\n#     # load paper\n#     paper = papers[paper_id]\n    \n#     # extract sentences\n#     sentences = [clean_training_text(sentence) for section in paper \n#                  for sentence in section['text'].split('.')\n#                 ]\n#     sentences = shorten_sentences(sentences) # make sentences short\n#     sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n#     sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n#     # collect all sentences in json\n#     for sentence in sentences:\n#         sentence_words = sentence.split()\n#         dummy_tags = ['O']*len(sentence_words)\n#         test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n\n#     # track which sentence belongs to which data point\n#     paper_length.append(len(sentences))\n\n# print(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:44.090286Z","iopub.execute_input":"2021-06-04T08:29:44.090566Z","iopub.status.idle":"2021-06-04T08:29:44.098967Z","shell.execute_reply.started":"2021-06-04T08:29:44.090534Z","shell.execute_reply":"2021-06-04T08:29:44.098052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in tqdm(train_test['Id'].unique()):\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:29:44.101678Z","iopub.execute_input":"2021-06-04T08:29:44.101941Z","iopub.status.idle":"2021-06-04T08:30:07.130952Z","shell.execute_reply.started":"2021-06-04T08:29:44.101917Z","shell.execute_reply":"2021-06-04T08:30:07.130230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:30:07.132148Z","iopub.execute_input":"2021-06-04T08:30:07.132479Z","iopub.status.idle":"2021-06-04T08:30:07.139199Z","shell.execute_reply.started":"2021-06-04T08:30:07.132445Z","shell.execute_reply":"2021-06-04T08:30:07.137259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:30:07.141052Z","iopub.execute_input":"2021-06-04T08:30:07.141519Z","iopub.status.idle":"2021-06-04T08:30:07.805147Z","shell.execute_reply.started":"2021-06-04T08:30:07.141467Z","shell.execute_reply":"2021-06-04T08:30:07.804128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:30:07.808899Z","iopub.execute_input":"2021-06-04T08:30:07.809166Z","iopub.status.idle":"2021-06-04T08:30:07.816282Z","shell.execute_reply.started":"2021-06-04T08:30:07.809138Z","shell.execute_reply":"2021-06-04T08:30:07.815223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(test_rows))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:30:07.817789Z","iopub.execute_input":"2021-06-04T08:30:07.818276Z","iopub.status.idle":"2021-06-04T08:30:07.824682Z","shell.execute_reply.started":"2021-06-04T08:30:07.818238Z","shell.execute_reply":"2021-06-04T08:30:07.823832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:30:07.826112Z","iopub.execute_input":"2021-06-04T08:30:07.826486Z","iopub.status.idle":"2021-06-04T08:48:47.908095Z","shell.execute_reply.started":"2021-06-04T08:30:07.826452Z","shell.execute_reply":"2021-06-04T08:48:47.906838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:47.911660Z","iopub.execute_input":"2021-06-04T08:48:47.911952Z","iopub.status.idle":"2021-06-04T08:48:48.014145Z","shell.execute_reply.started":"2021-06-04T08:48:47.911921Z","shell.execute_reply":"2021-06-04T08:48:48.013351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    # for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n    for sentence, pred in zip(test_sentences[:2], bert_outputs[:2]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.015527Z","iopub.execute_input":"2021-06-04T08:48:48.015879Z","iopub.status.idle":"2021-06-04T08:48:48.477487Z","shell.execute_reply.started":"2021-06-04T08:48:48.015852Z","shell.execute_reply":"2021-06-04T08:48:48.476564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.478846Z","iopub.execute_input":"2021-06-04T08:48:48.479213Z","iopub.status.idle":"2021-06-04T08:48:48.485202Z","shell.execute_reply.started":"2021-06-04T08:48:48.479175Z","shell.execute_reply":"2021-06-04T08:48:48.484154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.486673Z","iopub.execute_input":"2021-06-04T08:48:48.487024Z","iopub.status.idle":"2021-06-04T08:48:48.505026Z","shell.execute_reply.started":"2021-06-04T08:48:48.486987Z","shell.execute_reply":"2021-06-04T08:48:48.504205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.506676Z","iopub.execute_input":"2021-06-04T08:48:48.507043Z","iopub.status.idle":"2021-06-04T08:48:48.514594Z","shell.execute_reply.started":"2021-06-04T08:48:48.507009Z","shell.execute_reply":"2021-06-04T08:48:48.513551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"final_predictions = filtered_bert_labels\n# for literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n#     if literal_match:\n#         final_predictions.append(literal_match)\n#     else:\n#         final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.516212Z","iopub.execute_input":"2021-06-04T08:48:48.516716Z","iopub.status.idle":"2021-06-04T08:48:48.522739Z","shell.execute_reply.started":"2021-06-04T08:48:48.516679Z","shell.execute_reply":"2021-06-04T08:48:48.522077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission['PredictionString'] = final_predictions\n# sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.524016Z","iopub.execute_input":"2021-06-04T08:48:48.524464Z","iopub.status.idle":"2021-06-04T08:48:48.531226Z","shell.execute_reply.started":"2021-06-04T08:48:48.524428Z","shell.execute_reply":"2021-06-04T08:48:48.530491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['dataset_title2'] = [clean_training_text(x.lower()) for x in train['dataset_title']]\ntrain_test['dataset_title2'] = [clean_training_text(x.lower()) for x in train_test['dataset_title']]\n\ncomplete = pd.concat([train, train_test])\n\ndatalabels = pd.DataFrame()\ndatalabels['Id'] = complete['dataset_title2'].unique()\ndatalabels['Labels'] = [set(\n    [clean_training_text(x.lower()) for x in complete[complete['dataset_title'] == label]['dataset_label'].unique()] +\n    [clean_training_text(x.lower()) for x in complete[complete['dataset_title'] == label]['dataset_title'].unique()] +\n    [clean_training_text(x.lower()) for x in complete[complete['dataset_title'] == label]['cleaned_label'].unique()]) \n                                  for label in complete['dataset_title'].unique()]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:48.532261Z","iopub.execute_input":"2021-06-04T08:48:48.533348Z","iopub.status.idle":"2021-06-04T08:48:50.552879Z","shell.execute_reply.started":"2021-06-04T08:48:48.533310Z","shell.execute_reply":"2021-06-04T08:48:50.552107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uniq = train_test['Id'].unique() #train_test.groupby(['Id'])\n\ngroup_id = train_test.groupby(['Id'])\n\nt_new = pd.DataFrame(columns=['Id', 'Predictions'])\nfor u, pred in tqdm(zip(uniq, final_predictions)):\n    #rows = group_id.get_group(u)\n    predictions = list(filter(None, pred.split('|')))\n\n    label_predictions = []\n    for row, labels in datalabels.iterrows():\n        for label in labels['Labels']:\n            if label in predictions:\n                label_predictions.append(labels['Id'])\n\n    t_new = t_new.append({'Id': u, 'Predictions': label_predictions}, ignore_index=True)\n\n\n# t1 = []\n# for i in t_new['Predictions']:\n#     for j in i:\n#         t1.append(j)\n# t1 = list(set(t1))\n\n# t2 = []\n# for i in train_test['dataset_title']:\n#     t2.append(clean_training_text(i.lower()))\n# t2 = list(set(t2))\n# # t_new['Predictions'].head()\n# print(sorted(t1)[:5])\n# print(sorted(t2)[:5])\nt_new.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:48:50.554174Z","iopub.execute_input":"2021-06-04T08:48:50.554515Z","iopub.status.idle":"2021-06-04T08:50:17.851462Z","shell.execute_reply.started":"2021-06-04T08:48:50.554464Z","shell.execute_reply":"2021-06-04T08:50:17.850508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(train_test[:5])\n# print(\"test\")\n# print(train[:5])\n# train_test['Id'].unique()[:5]\n#[len(i) for i in t_new['Predictions'][:5]]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:17.853113Z","iopub.execute_input":"2021-06-04T08:50:17.853455Z","iopub.status.idle":"2021-06-04T08:50:17.857457Z","shell.execute_reply.started":"2021-06-04T08:50:17.853419Z","shell.execute_reply":"2021-06-04T08:50:17.856551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.to_csv(f'submission.csv', index=False)\n# t_new.to_csv(f'submission.csv', index=False)\n# Verschillende namen voor datasets","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:17.858862Z","iopub.execute_input":"2021-06-04T08:50:17.859239Z","iopub.status.idle":"2021-06-04T08:50:17.869752Z","shell.execute_reply.started":"2021-06-04T08:50:17.859202Z","shell.execute_reply":"2021-06-04T08:50:17.868908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:17.874405Z","iopub.execute_input":"2021-06-04T08:50:17.874696Z","iopub.status.idle":"2021-06-04T08:50:17.882028Z","shell.execute_reply.started":"2021-06-04T08:50:17.874671Z","shell.execute_reply":"2021-06-04T08:50:17.880960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check how much we are matching #train_df\ncorrect, correct2, wrong_assign, wrong_miss_test, wrong_miss_train = 0, 0, 0, 0, 0\nempty = 0\n\n# Check all predictions\nfor _, data in tqdm(t_new.iterrows()):\n    # No predictions\n    if len(data['Predictions']) == 0:\n        empty += 1\n    for label in data['Predictions']:\n        # Correct prediction\n        if ((train_test['Id']==data['Id']) & (train_test['dataset_title2']==label)).any():\n            correct += 1\n        # Prediction is in training set\n        elif ((train['Id']==data['Id']) & (train['dataset_title2']==label)).any():\n            correct2 += 1\n        # Prediction is not in test or training set\n        else:\n            #In labels, but not test set likely comes from bad train value\n            wrong_assign += 1\n\n# Check if match with test set\nfor _, data in tqdm(train_test.iterrows()):\n    check = True\n    for _, sub in t_new.loc[t_new['Id']==data['Id']].iterrows():\n        if (data['dataset_title2'] in sub['Predictions']):\n            check = False\n            break\n\n    # Prediction is missing from test_set possibly\n    if (check):\n        wrong_miss_test += 1\n\n# Check if match with training set\nfor _, data in tqdm(train.iterrows()):\n    check = True\n    for _, sub in t_new.loc[t_new['Id']==data['Id']].iterrows():\n        if (data['dataset_title2'] in sub['Predictions']):\n            check = False\n            break\n\n    # Prediction is missing from training_set\n    if ((t_new['Id']==data['Id']).any() and check):\n        wrong_miss_train += 1\n\n\nprint(f\"Correct: {correct}, Correct2: {correct2}, Wrongly assigned: {wrong_assign}, Missed test: {wrong_miss_test}, Missed train: {wrong_miss_train}, Empty: {empty} \")","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:17.883710Z","iopub.execute_input":"2021-06-04T08:50:17.884162Z","iopub.status.idle":"2021-06-04T08:50:43.666694Z","shell.execute_reply.started":"2021-06-04T08:50:17.884098Z","shell.execute_reply":"2021-06-04T08:50:43.665185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BOTTOM SET\ntitles_sorted = complete.groupby('dataset_title') \\\n                        .count() \\\n                        .reset_index() \\\n                        .sort_values(['Id'], ascending=True)\n\ntitles_selected = []\ncount = 0\nfor row, data in titles_sorted.iterrows():\n    titles_selected.append(data['dataset_title'])\n    count += data['Id']\n    if count >= len(complete) * 0.2:\n        break\n\nmsk = np.array([(x in titles_selected) for x in train_test['dataset_title']])\ncheck_test = train_test[msk]\n\nmsk = np.array([(x in titles_selected) for x in train['dataset_title']])\ncheck_train = train[~msk]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:43.668004Z","iopub.execute_input":"2021-06-04T08:50:43.668352Z","iopub.status.idle":"2021-06-04T08:50:43.761104Z","shell.execute_reply.started":"2021-06-04T08:50:43.668316Z","shell.execute_reply":"2021-06-04T08:50:43.760270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check how much we are matching #train_df\ncorrect, correct2, wrong_assign, wrong_miss_test, wrong_miss_train = 0, 0, 0, 0, 0\nempty = 0\ncheck_temp = 0\n\n# Check all predictions\nfor _, data in tqdm(t_new.iterrows()):\n    # No predictions\n    if len(data['Predictions']) == 0:\n        empty += 1\n    for label in data['Predictions']:\n        # Correct prediction\n        if ((check_test['Id']==data['Id']) & (check_test['dataset_title2']==label)).any():\n            correct += 1\n        # Prediction is in training set\n        elif ((check_train['Id']==data['Id']) & (check_train['dataset_title2']==label)).any():\n            correct2 += 1\n        # Prediction is not in test or training set\n        else:\n            #In labels, but not test set likely comes from bad train value\n            wrong_assign += 1\n\n# Check if match with test set\nfor _, data in tqdm(check_test.iterrows()):\n    check = True\n    for _, sub in t_new.loc[t_new['Id']==data['Id']].iterrows():\n        if (data['dataset_title2'] in sub['Predictions']):\n            check = False\n            break\n\n    # Prediction is missing from test_set possibly\n    if (check):\n        wrong_miss_test += 1\n\n# Check if match with training set\nfor _, data in tqdm(check_train.iterrows()):\n    check = True\n    for _, sub in t_new.loc[t_new['Id']==data['Id']].iterrows():\n        if (data['dataset_title2'] in sub['Predictions']):\n            check = False\n            break\n\n    # Prediction is missing from training_set\n    if ((t_new['Id']==data['Id']).any() and check):\n        wrong_miss_train += 1\n\n\nprint(f\"Correct: {correct}, Correct2: {correct2}, Wrongly assigned: {wrong_assign}, Missed test: {wrong_miss_test}, Missed train: {wrong_miss_train}, Empty: {empty} \")","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:50:43.763388Z","iopub.execute_input":"2021-06-04T08:50:43.763747Z","iopub.status.idle":"2021-06-04T08:51:02.497624Z","shell.execute_reply.started":"2021-06-04T08:50:43.763711Z","shell.execute_reply":"2021-06-04T08:51:02.496929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(check_train))\n[print(f\"{col}:{len(check_train[col].unique())}\") for col in check_train.columns]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T08:55:17.420883Z","iopub.execute_input":"2021-06-04T08:55:17.421224Z","iopub.status.idle":"2021-06-04T08:55:17.446090Z","shell.execute_reply.started":"2021-06-04T08:55:17.421193Z","shell.execute_reply":"2021-06-04T08:55:17.445212Z"},"trusted":true},"execution_count":null,"outputs":[]}]}