{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:42:44.735475Z","iopub.execute_input":"2021-06-05T10:42:44.735983Z","iopub.status.idle":"2021-06-05T10:44:05.810616Z","shell.execute_reply.started":"2021-06-05T10:42:44.735942Z","shell.execute_reply":"2021-06-05T10:44:05.809432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)\n\nMAX_SAMPLE = None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-05T10:44:05.812394Z","iopub.execute_input":"2021-06-05T10:44:05.812724Z","iopub.status.idle":"2021-06-05T10:44:05.818189Z","shell.execute_reply.started":"2021-06-05T10:44:05.812693Z","shell.execute_reply":"2021-06-05T10:44:05.817373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df = train_df[:MAX_SAMPLE]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:05.819139Z","iopub.execute_input":"2021-06-05T10:44:05.819427Z","iopub.status.idle":"2021-06-05T10:44:05.908412Z","shell.execute_reply.started":"2021-06-05T10:44:05.819398Z","shell.execute_reply":"2021-06-05T10:44:05.907407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train_df['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:05.909622Z","iopub.execute_input":"2021-06-05T10:44:05.90987Z","iopub.status.idle":"2021-06-05T10:44:16.527791Z","shell.execute_reply.started":"2021-06-05T10:44:05.909845Z","shell.execute_reply":"2021-06-05T10:44:16.526706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.530805Z","iopub.execute_input":"2021-06-05T10:44:16.531081Z","iopub.status.idle":"2021-06-05T10:44:16.540036Z","shell.execute_reply.started":"2021-06-05T10:44:16.531053Z","shell.execute_reply":"2021-06-05T10:44:16.539166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.543241Z","iopub.execute_input":"2021-06-05T10:44:16.543657Z","iopub.status.idle":"2021-06-05T10:44:16.548793Z","shell.execute_reply.started":"2021-06-05T10:44:16.543626Z","shell.execute_reply":"2021-06-05T10:44:16.54782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame()\n\nfor test_file in test_files:\n    text_data=pd.read_json(test_file)\n    text_data.insert(0,'id', test_file.split('/')[-1].split('.')[0]) \n    test_df = pd.concat([test_df, text_data])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.55017Z","iopub.execute_input":"2021-06-05T10:44:16.550417Z","iopub.status.idle":"2021-06-05T10:44:16.58631Z","shell.execute_reply.started":"2021-06-05T10:44:16.550392Z","shell.execute_reply":"2021-06-05T10:44:16.585518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = set()\n\nfor label1, label2, label3 in train_df[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label1).lower())\n    all_labels.add(str(label2).lower())\n    all_labels.add(str(label3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.589182Z","iopub.execute_input":"2021-06-05T10:44:16.589428Z","iopub.status.idle":"2021-06-05T10:44:16.630166Z","shell.execute_reply.started":"2021-06-05T10:44:16.589404Z","shell.execute_reply":"2021-06-05T10:44:16.629517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bigger_dataset_df = pd.read_csv('../input/filtered-bigger-govt-dataset/ExtraLabelsCleaned.txt')\nbigger_dataset_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.631134Z","iopub.execute_input":"2021-06-05T10:44:16.631539Z","iopub.status.idle":"2021-06-05T10:44:16.645037Z","shell.execute_reply.started":"2021-06-05T10:44:16.631501Z","shell.execute_reply":"2021-06-05T10:44:16.643978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label in bigger_dataset_df['Label']:\n    all_labels.add(str(label).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.646151Z","iopub.execute_input":"2021-06-05T10:44:16.646394Z","iopub.status.idle":"2021-06-05T10:44:16.65088Z","shell.execute_reply.started":"2021-06-05T10:44:16.64637Z","shell.execute_reply":"2021-06-05T10:44:16.649923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.652011Z","iopub.execute_input":"2021-06-05T10:44:16.652289Z","iopub.status.idle":"2021-06-05T10:44:16.664018Z","shell.execute_reply.started":"2021-06-05T10:44:16.652263Z","shell.execute_reply":"2021-06-05T10:44:16.663407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['cleaned_text'] = test_df.apply(lambda txt: clean_text(txt['text']).strip(), axis =1)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.665485Z","iopub.execute_input":"2021-06-05T10:44:16.665993Z","iopub.status.idle":"2021-06-05T10:44:16.701777Z","shell.execute_reply.started":"2021-06-05T10:44:16.665956Z","shell.execute_reply":"2021-06-05T10:44:16.700977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_group = pd.DataFrame(test_df.groupby(['id'])['cleaned_text'].agg(' '.join))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:16.702986Z","iopub.execute_input":"2021-06-05T10:44:16.703527Z","iopub.status.idle":"2021-06-05T10:44:16.713196Z","shell.execute_reply.started":"2021-06-05T10:44:16.703484Z","shell.execute_reply":"2021-06-05T10:44:16.712389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = test_group['cleaned_text'].apply(lambda txt: '|'.join({label for label in all_labels if label in txt})).reset_index()\nresult.columns =['Id', 'PredictionString']","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:29.00647Z","iopub.execute_input":"2021-06-05T10:44:29.006819Z","iopub.status.idle":"2021-06-05T10:44:29.059054Z","shell.execute_reply.started":"2021-06-05T10:44:29.00678Z","shell.execute_reply":"2021-06-05T10:44:29.058204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor pred in result['PredictionString']:\n    literal_preds.append(pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:31.777218Z","iopub.execute_input":"2021-06-05T10:44:31.777536Z","iopub.status.idle":"2021-06-05T10:44:31.78193Z","shell.execute_reply.started":"2021-06-05T10:44:31.777507Z","shell.execute_reply":"2021-06-05T10:44:31.781142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:35.437864Z","iopub.execute_input":"2021-06-05T10:44:35.438358Z","iopub.status.idle":"2021-06-05T10:44:35.444575Z","shell.execute_reply.started":"2021-06-05T10:44:35.438315Z","shell.execute_reply":"2021-06-05T10:44:35.443402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64 \nOVERLAP = 20 \n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/coleridge-bert-models/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/coleridge-bert-models/train_ner.json'\nVAL_PATH = '../input/coleridge-bert-models/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:41.163748Z","iopub.execute_input":"2021-06-05T10:44:41.164071Z","iopub.status.idle":"2021-06-05T10:44:41.169158Z","shell.execute_reply.started":"2021-06-05T10:44:41.164041Z","shell.execute_reply":"2021-06-05T10:44:41.16795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train_df)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:44.395031Z","iopub.execute_input":"2021-06-05T10:44:44.395516Z","iopub.status.idle":"2021-06-05T10:44:44.803311Z","shell.execute_reply.started":"2021-06-05T10:44:44.395485Z","shell.execute_reply":"2021-06-05T10:44:44.802265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:48.007942Z","iopub.execute_input":"2021-06-05T10:44:48.008272Z","iopub.status.idle":"2021-06-05T10:44:48.013686Z","shell.execute_reply.started":"2021-06-05T10:44:48.008244Z","shell.execute_reply":"2021-06-05T10:44:48.01283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in result['Id']:\n\n    paper = papers[paper_id]\n    \n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:51.302254Z","iopub.execute_input":"2021-06-05T10:44:51.302573Z","iopub.status.idle":"2021-06-05T10:44:51.339497Z","shell.execute_reply.started":"2021-06-05T10:44:51.302546Z","shell.execute_reply":"2021-06-05T10:44:51.338537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:54.670592Z","iopub.execute_input":"2021-06-05T10:44:54.671104Z","iopub.status.idle":"2021-06-05T10:44:54.675661Z","shell.execute_reply.started":"2021-06-05T10:44:54.67106Z","shell.execute_reply":"2021-06-05T10:44:54.674795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:44:58.022617Z","iopub.execute_input":"2021-06-05T10:44:58.023111Z","iopub.status.idle":"2021-06-05T10:44:58.790739Z","shell.execute_reply.started":"2021-06-05T10:44:58.023068Z","shell.execute_reply":"2021-06-05T10:44:58.789321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:45:01.219808Z","iopub.execute_input":"2021-06-05T10:45:01.220151Z","iopub.status.idle":"2021-06-05T10:45:01.224964Z","shell.execute_reply.started":"2021-06-05T10:45:01.220111Z","shell.execute_reply":"2021-06-05T10:45:01.224033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    !rm -r \"$OUTPUT_DIR\"\n    \n    bert_predict()\n    \n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:45:03.962894Z","iopub.execute_input":"2021-06-05T10:45:03.963335Z","iopub.status.idle":"2021-06-05T10:46:59.969782Z","shell.execute_reply.started":"2021-06-05T10:45:03.963306Z","shell.execute_reply":"2021-06-05T10:46:59.968336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:03.061244Z","iopub.execute_input":"2021-06-05T10:47:03.061678Z","iopub.status.idle":"2021-06-05T10:47:03.068771Z","shell.execute_reply.started":"2021-06-05T10:47:03.061637Z","shell.execute_reply":"2021-06-05T10:47:03.067767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] \n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': \n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: \n                curr_phrase += ' ' + word\n            else: \n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n       \n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:05.436895Z","iopub.execute_input":"2021-06-05T10:47:05.437208Z","iopub.status.idle":"2021-06-05T10:47:05.448105Z","shell.execute_reply.started":"2021-06-05T10:47:05.437181Z","shell.execute_reply":"2021-06-05T10:47:05.447209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:16.61972Z","iopub.execute_input":"2021-06-05T10:47:16.620109Z","iopub.status.idle":"2021-06-05T10:47:16.626069Z","shell.execute_reply.started":"2021-06-05T10:47:16.62008Z","shell.execute_reply":"2021-06-05T10:47:16.625456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:20.124453Z","iopub.execute_input":"2021-06-05T10:47:20.124928Z","iopub.status.idle":"2021-06-05T10:47:20.131228Z","shell.execute_reply.started":"2021-06-05T10:47:20.124898Z","shell.execute_reply":"2021-06-05T10:47:20.129941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:23.186907Z","iopub.execute_input":"2021-06-05T10:47:23.18723Z","iopub.status.idle":"2021-06-05T10:47:23.191716Z","shell.execute_reply.started":"2021-06-05T10:47:23.187202Z","shell.execute_reply":"2021-06-05T10:47:23.191126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\nfor literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n    if literal_match:\n        final_predictions.append(literal_match)\n    else:\n        final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:27.134563Z","iopub.execute_input":"2021-06-05T10:47:27.135114Z","iopub.status.idle":"2021-06-05T10:47:27.139003Z","shell.execute_reply.started":"2021-06-05T10:47:27.13508Z","shell.execute_reply":"2021-06-05T10:47:27.138196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['PredictionString'] = final_predictions\nresult.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:30.348313Z","iopub.execute_input":"2021-06-05T10:47:30.348672Z","iopub.status.idle":"2021-06-05T10:47:30.365114Z","shell.execute_reply.started":"2021-06-05T10:47:30.348643Z","shell.execute_reply":"2021-06-05T10:47:30.364327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:47:34.571139Z","iopub.execute_input":"2021-06-05T10:47:34.571616Z","iopub.status.idle":"2021-06-05T10:47:34.582505Z","shell.execute_reply.started":"2021-06-05T10:47:34.571584Z","shell.execute_reply":"2021-06-05T10:47:34.58143Z"},"trusted":true},"execution_count":null,"outputs":[]}]}