{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Abstract","metadata":{}},{"cell_type":"markdown","source":"Extract the dataset name included in the treatise by NER. <br>\n\nSince data names rarely appear across sentences, modeling by dividing them into sentence units instead of paper units.","metadata":{}},{"cell_type":"markdown","source":"# Reference","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/tungmphung/coleridge-matching-bert-ner <br>\nhttps://github.com/huggingface/transformers/blob/master/examples/README.md","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"# ============== Setup Module ============== #\n!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Import Module ============== #\nimport os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\nimport gc\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Set Constance ============== #\nTRAIN_DATA_PATH = '../input/coleridgeinitiative-show-us-the-data/train.csv'\nSUBMISSION_PATH = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\n\nPAPER_TRAIN_FOLDER = '../input/coleridgeinitiative-show-us-the-data/train'\nPAPER_TEST_FOLDER = '../input/coleridgeinitiative-show-us-the-data/test'\nRICH_CONTEXT = '../input/coleridge-intiative-rich-context/train_test'\n\nPAPER_DEVIDE_SENT_PATH = '../input/coleride/train_literal_label.pickle'\n# PAPER_DEVIDE_SENT_PATH = ''\n\nIS_TRAIN = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"code","source":"# ============== NLP Preprocess Helper ============== #\ndef paper_to_concate_json(id_list, folder):\n    papers = {}\n    for paper_id in tqdm(id_list):\n        with open(f'{folder}/{paper_id}.json', 'r') as f:\n            paper = json.load(f)\n            papers[paper_id] = paper\n    return papers\n\ndef clean_text(txt, is_lower=True):\n    if is_lower:\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n    else:\n        return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef totally_clean_text(txt, is_lower=True):\n    txt = clean_text(txt, is_lower)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef devide_sentence(id_list, papers, out_file):\n    literal_pdf = pd.DataFrame()\n    \n    for paper_id in tqdm(id_list):\n        paper = papers[paper_id]\n        text = pd.DataFrame([totally_clean_text(sent) for section in paper for sent in section[\"text\"].split(\". \") if len(sent) > 0],columns=[\"sent\"])\n        text[\"Id\"] = paper_id\n        literal_pdf = pd.concat([literal_pdf, text], axis=0)\n    literal_pdf.to_pickle(f\"{out_file}.pickle\")\n    return literal_pdf\n    \ndef is_in_label(labels, in_file, out_file):\n    literal_pdf = pd.read_pickle(f\"{in_file}.pickle\")\n    label_pdf = pd.DataFrame()\n    for label in tqdm(labels):\n        literal_pdf[label] = literal_pdf[\"sent\"].str.contains(label) * 1\n        \n        tmp = literal_pdf[literal_pdf[label] == 1][[\"Id\", \"sent\", label]]\n        tmp[\"Label\"] = label\n        label_pdf = pd.concat([label_pdf, tmp[[\"Id\", \"sent\", \"Label\"]]])\n        gc.collect()\n        \n    label_pdf.to_pickle(f\"{out_file}.pickle\")\n    return label_pdf\n\ndef change_enr_format(id_list, papers, labels, out_file, is_train_data=True):\n    if is_train_data:\n        devide_sentence(id_list, papers, \"train_literal\")\n        literal_pdf = is_in_label(labels, \"train_literal\", out_file)\n        \n    else:\n        literal_pdf = devide_sentence(id_list, papers, \"test_literal\")\n    \n    return literal_pdf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Load Data ============== #\ntrain_df = pd.read_csv(TRAIN_DATA_PATH)\nsub_df   = pd.read_csv(SUBMISSION_PATH)\n\n## Concate Paper\ntrain_id_list = train_df[\"Id\"].unique()\ntest_id_list = sub_df[\"Id\"].unique()\n# train_paper = paper_to_concate_json(train_id_list, PAPER_TRAIN_FOLDER)\ntest_paper = paper_to_concate_json(test_id_list, PAPER_TEST_FOLDER)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Get Train Label(cleaned) ============== #\nall_labels = set()\n\nfor label_1, label_2, label_3 in train_df[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(clean_text(label_1))\n    all_labels.add(clean_text(label_2))\n    all_labels.add(clean_text(label_3))\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Paper Devide Sentense ============== #\n\n## Coleridge Initiative\n# Train\nif not os.path.exists(PAPER_DEVIDE_SENT_PATH):\n    train_literal_df = change_enr_format(train_id_list, train_paper, all_labels, \"train_literal_label\")\nelse:\n    train_literal_df = pd.read_pickle(PAPER_DEVIDE_SENT_PATH)\n\n# Test\ntest_literal_df = change_enr_format(test_id_list, test_paper, None, \"test_literal_label\", is_train_data=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(os.path.join(f\"{RICH_CONTEXT}/data_set_citations.json\")) as f:\n    rich_citations = json.load(f)\n\nmentions_list = []\nfor rc in tqdm(rich_citations):\n    pub_id = rc[\"publication_id\"]\n    for mentions in rc['mention_list']:\n        mentions_list.append([pub_id, mentions])\nmentions_df = pd.DataFrame(mentions_list, columns=[\"pub_id\", \"mentions\"])\nmentions_df[\"cleaned_mention\"] = mentions_df[\"mentions\"].apply(lambda x: clean_text(x, is_lower=False))\n\nsent_list = []\nfor ids in tqdm(mentions_df[\"pub_id\"].unique()):\n    path = f\"{RICH_CONTEXT}/files/text/{ids}.txt\"\n    with open(path, \"r\") as f:\n        lines = f.readlines()\n    \n    lines = \"\".join(lines)\n    lines = lines.replace(\"\\n\", \" \")\n    sentences = lines.split(\". \")\n    for sentence in sentences:\n        clean_sent = totally_clean_text(sentence, is_lower=False)\n        sent_list.append([ids, clean_sent])\nsent_df = pd.DataFrame(sent_list, columns=[\"Id\", \"sent\"])\n\nadd_train_df = pd.DataFrame()\nfor ids in tqdm(mentions_df[\"pub_id\"].unique()):\n    mention_list = mentions_df[mentions_df[\"pub_id\"] == ids][\"cleaned_mention\"].tolist()\n    sentense_list = sent_df[sent_df[\"Id\"] == ids].reset_index(drop=True)\n    for mention in mention_list:\n        tmp = sentense_list[sentense_list[\"sent\"].str.contains(mention)].reset_index(drop=True)\n        tmp[\"Label\"] = mention\n        add_train_df =  pd.concat([add_train_df, tmp[[\"Id\", \"sent\", \"Label\"]]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Checking datasets that cannot be retrieved ============== #\n\n# 19661\ntrain_df[\"cleaned_label\"] = train_df[\"cleaned_label\"].str.rstrip()\ntmp = train_df.merge(train_literal_df[[\"Id\", \"Label\"]].drop_duplicates(), left_on=[\"Id\", \"cleaned_label\"], right_on=[\"Id\", \"Label\"], how=\"left\")\n\n# 6\ntmp[tmp[\"Label\"].isnull()]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Paper Id: cf2aaa14-bd90-4e69-aca4-c2855747b0e5 <br>\nPaper Id: 44339037-7785-4d4d-b334-b47fd81b8b9e\n* Straddle sentences\n* Ex: National Science Foundation. Survey of Earned Doctorates\n\nPaper Id: 45dd2256-74f2-4f72-beb2-a8b770baf233\n* Original data name contains dots\n* Ex: NSF. Survey of Earned Doctorates\n\nPaper Id: cb21f8af-8296-4970-ad64-24821f2eeb61 <br>\nPaper Id: a69f443d-6318-40ef-aa0e-b08c2ec338f8 <br>\nPaper Id: 29b4a5a2-1304-4a22-8a14-c2aebae5503c\n* Dataset name and Reference Number?\n* Ex: (SARS-CoV) [2] . Genome sequences","metadata":{}},{"cell_type":"code","source":"add_train_df.columns = [\"Id\", \"sent\", \"Label\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Add NER Label (Train) ============== #\n# train_literal_df = pd.concat([train_literal_df[[\"Id\", \"sent\", \"Label\"]], add_train_df])\ntrain_rows_dict = {}\nfor ids, sents, label in tqdm(add_train_df[[\"Id\", \"sent\", \"Label\"]].itertuples(index=False)):\n    sent_list = sents.split()\n    label_sent_list = label.split()\n    \n    if not ids in train_rows_dict:\n        train_rows_dict[ids] = []\n    \n    dummy_tags = []\n    idx = 0\n    for i, sent in enumerate(sent_list):\n        if sent == label_sent_list[idx]:\n            idx += 1\n            if idx == len(label_sent_list):\n                dummy_tags += [\"B\"] + [\"I\"] * (idx - 1)\n                idx = 0\n            else:\n                if i == (len(sent_list) - 1):\n                    dummy_tags.extend([\"O\"] * (idx))\n                    idx = 0\n        else:\n            dummy_tags.extend([\"O\"] * (idx + 1))\n            idx = 0\n            \n\n    assert len(sent_list) == len(dummy_tags)\n    train_rows_dict[ids].append({'tokens' : sent_list, 'tags' : dummy_tags})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_in_ids = list(train_rows_dict.keys())\n# set(train_id_list) - set(label_in_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Seperate Train Valid ============== #\ntrain_id_count = len(label_in_ids)\nsel_train_id = np.random.choice(label_in_ids, int(train_id_count * 0.8))\n\ntrain_rows = []\nvalid_rows = []\nfor ids in tqdm(label_in_ids):\n    if ids in sel_train_id:\n        train_rows += train_rows_dict[ids]\n    else:\n        valid_rows += train_rows_dict[ids]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Add NER Label (Test)============== #\n# test_rows = []\n# for sents in tqdm(test_literal_df[\"sent\"]):\n#     sent_list = sents.split()\n#     dummy_tags = [\"O\"] * len(sent_list)\n            \n#     test_rows.append({'tokens' : sent_list, 'tags' : dummy_tags})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Add NER Label (Train All)============== #\n# train_literal_all_df = pd.read_pickle(\"../input/coleride/train_literal.pickle\")\n# train_literal_all_df = train_literal_all_df.sample(frac=0.1)\n# train_all_rows = []\n# for sents in tqdm(train_literal_all_df[\"sent\"]):\n#     sent_list = sents.split()\n#     dummy_tags = [\"O\"] * len(sent_list)\n            \n#     train_all_rows.append({'tokens' : sent_list, 'tags' : dummy_tags})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rows[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"# ============== Set Constance (Model)============== #\n\nMAX_LENGTH = 256\nOVERLAP = 20 \n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '.'\nTEST_INPUT_SAVE_PATH = './test'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_INPUT_SAVE_PATH = './train'\nTRAIN_PATH = 'train_ner.json'\nVAL_PATH = 'valid_ner.json'\n\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Set Environ ============== #\nos.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_INPUT_SAVE_PATH}/{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{TRAIN_INPUT_SAVE_PATH}/{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Create Directory ============== #\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)\n\n# make necessart directories and files\nos.makedirs(TRAIN_INPUT_SAVE_PATH, exist_ok=True)\n\n# make necessart directories and files\nos.makedirs(PREDICTION_SAVE_PATH, exist_ok=True)\n\n# make necessart directories and files\nos.makedirs(PRETRAINED_PATH, exist_ok=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Train ============== #\ndef bert_train():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path bert-base-cased \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --num_train_epochs 5 \\\n    --per_device_train_batch_size 16 \\\n    --per_device_eval_batch_size 16 \\\n    --save_steps 15000 \\\n    --output_dir \"$MODEL_PATH\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_train \\\n    --do_eval \\\n    --overwrite_output_dir","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Pred ============== #\ndef bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Execute Train ============== #\n\n# for batch_begin in range(0, len(train_rows), PREDICT_BATCH):\nwith open(f'{TRAIN_INPUT_SAVE_PATH}/{TRAIN_PATH}', 'w') as f:\n    for row in train_rows:\n        json.dump(row, f)\n        f.write('\\n')\n\nwith open(f'{TRAIN_INPUT_SAVE_PATH}/{VAL_PATH}', 'w') as f:\n    for row in valid_rows:\n        json.dump(row, f)\n        f.write('\\n')\n\n# if IS_TRAIN:\nbert_train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"# ============== Execute Predict ============== #\n\nbert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ============== Get NER ============== #\ndef get_ner(literal_df, rows, bert_outputs):\n    labels = []\n    for ids, sentence, pred in zip(literal_df[\"Id\"], rows, bert_outputs):\n        curr_phrase = ''\n        for word, tag in zip(sentence[\"tokens\"], pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.append([ids, curr_phrase])\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.append([ids, curr_phrase])\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.append([ids, curr_phrase])\n            curr_phrase = ''\n\n    pred_subs = pd.DataFrame(labels, columns=[\"Id\", \"Label\"]).drop_duplicates()\n    return pred_subs\npred_subs = get_ner(test_literal_df, test_rows, bert_outputs)\n# pred_subs = get_ner(train_literal_all_df, train_all_rows, bert_outputs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_subs.to_csv(\"ner_list.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# ============== Submission (Handle Unknown Test Dataset) ============== #\n\npreds = pred_subs.groupby(\"Id\")[\"Label\"].apply(list).apply(lambda x: \"|\".join(x)).to_frame()\npreds.columns = [\"PredictionString\"]\nsubmission = pd.read_csv(SUBMISSION_PATH)\n\nid_list = []\npred_list = []\nfor ids in submission[\"Id\"].tolist():\n    id_list.append(ids)\n    if ids in preds.index:\n        pred_list.append(preds.loc[ids][\"PredictionString\"])\n    else:\n        pred_list.append(\"\")\n\nsubmission = pd.DataFrame({\"Id\": id_list, \"PredictionString\": pred_list})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}