{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\n# !pip3 uninstall fsspec -y\n!pip install fsspec==2021.5.0\n\nfrom IPython.display import clear_output\n\nclear_output()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T02:09:24.954279Z","iopub.execute_input":"2021-06-22T02:09:24.954594Z","iopub.status.idle":"2021-06-22T02:09:59.688222Z","shell.execute_reply.started":"2021-06-22T02:09:24.954522Z","shell.execute_reply":"2021-06-22T02:09:59.687337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:09:59.689977Z","iopub.execute_input":"2021-06-22T02:09:59.690341Z","iopub.status.idle":"2021-06-22T02:10:00.386028Z","shell.execute_reply.started":"2021-06-22T02:09:59.690301Z","shell.execute_reply":"2021-06-22T02:10:00.385180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:10:00.387657Z","iopub.execute_input":"2021-06-22T02:10:00.388020Z","iopub.status.idle":"2021-06-22T02:10:01.032226Z","shell.execute_reply.started":"2021-06-22T02:10:00.387984Z","shell.execute_reply":"2021-06-22T02:10:01.031163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:10:01.034609Z","iopub.execute_input":"2021-06-22T02:10:01.034881Z","iopub.status.idle":"2021-06-22T02:10:01.041896Z","shell.execute_reply.started":"2021-06-22T02:10:01.034843Z","shell.execute_reply":"2021-06-22T02:10:01.041101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\n# train = pd.read_csv(train_path)\n# train = train[:MAX_SAMPLE]\n# print(f'No. raw training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:15:07.811420Z","iopub.execute_input":"2021-06-22T02:15:07.811755Z","iopub.status.idle":"2021-06-22T02:15:07.868270Z","shell.execute_reply.started":"2021-06-22T02:15:07.811725Z","shell.execute_reply":"2021-06-22T02:15:07.867272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train.groupby('Id').agg({\n#     'pub_title': 'first',\n#     'dataset_title': '|'.join,\n#     'dataset_label': '|'.join,\n#     'cleaned_label': '|'.join\n# }).reset_index()\n\n# print(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:15:08.186798Z","iopub.execute_input":"2021-06-22T02:15:08.187159Z","iopub.status.idle":"2021-06-22T02:15:08.534543Z","shell.execute_reply.started":"2021-06-22T02:15:08.187126Z","shell.execute_reply":"2021-06-22T02:15:08.533622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load datasets","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:10:13.097230Z","iopub.execute_input":"2021-06-22T02:10:13.097552Z","iopub.status.idle":"2021-06-22T02:10:13.102261Z","shell.execute_reply.started":"2021-06-22T02:10:13.097523Z","shell.execute_reply":"2021-06-22T02:10:13.101063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp8 = pd.read_csv('../input/ci-ext-datasets-found-in-train-v2/train_ext_data.csv')\ntmp8['ext_cleaned_label'] = tmp8['ext_cleaned_label'].apply(lambda x: x.split('|'))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:39:25.779120Z","iopub.execute_input":"2021-06-22T02:39:25.779444Z","iopub.status.idle":"2021-06-22T02:39:25.843779Z","shell.execute_reply.started":"2021-06-22T02:39:25.779415Z","shell.execute_reply":"2021-06-22T02:39:25.842997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_datasets = ['about', 'climatologists', 'control', 'exploration', 'defense', \n                'american community', 'american landscape', 'current population survey',\n                'gulf of maine', 'argonne national laboratory s greet', \n                'annual wholesale trade',\n                'bird conservation areas', 'bird incidental take', 'new housing', 'business patterns',\n                'create', 'federal aid to states', 'freedom of information act', 'fruit and vegetable prices',\n                'guidance navigation and control', 'high school and beyond', 'human resource management', \n                'housing unit estimates', 'international data base', 'labor market analysts', 'major land uses',\n                'mars exploration program', 'new residential construction', 'oxygen delivery system',\n                'pilot boarding areas', 'profiles in science', 'state fact sheets', 'summary of business',\n                'tsunamis general', 'virtual grower', # 0.620\n               ]\nwrong_names = [\n    'national assessment of educational progress',\n    'national postsecondary student aid study',\n    'nursing home compare',\n    'private school universe survey',\n    'program for international student assessment',\n    'progress in international reading literacy study',\n    'schools and staffing survey'\n]\n\na = tmp8['ext_cleaned_label'].values\nb = []\nfor l in tqdm(a):\n    sub_label = []\n    for l2 in l:\n        append_ = True\n        for l3 in not_datasets:\n            if l3 in l2:\n                append_ = False\n        if append_:\n            for wn in wrong_names:\n                if wn in l2:\n                    l2 = wn\n            sub_label.append(l2)\n    b.append(sub_label)\ntmp8['ext_cleaned_label2'] = b\ntmp8['ext_cleaned_label2'] = tmp8['ext_cleaned_label2'].apply(lambda x: np.unique(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:39:26.085232Z","iopub.execute_input":"2021-06-22T02:39:26.085553Z","iopub.status.idle":"2021-06-22T02:39:26.424878Z","shell.execute_reply.started":"2021-06-22T02:39:26.085522Z","shell.execute_reply":"2021-06-22T02:39:26.424084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = tmp8\nprint(len(train))\ntmp8.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:39:28.784679Z","iopub.execute_input":"2021-06-22T02:39:28.785057Z","iopub.status.idle":"2021-06-22T02:39:28.810019Z","shell.execute_reply.started":"2021-06-22T02:39:28.785014Z","shell.execute_reply":"2021-06-22T02:39:28.809186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(train['Id'].unique()):\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:14:04.082918Z","iopub.execute_input":"2021-06-22T02:14:04.083301Z","iopub.status.idle":"2021-06-22T02:14:58.019012Z","shell.execute_reply.started":"2021-06-22T02:14:04.083255Z","shell.execute_reply":"2021-06-22T02:14:58.018118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        \n        big_list = [x.lower() for x in big_list]\n        \n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence.lower())\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:44:45.110971Z","iopub.execute_input":"2021-06-22T02:44:45.111301Z","iopub.status.idle":"2021-06-22T02:44:45.122069Z","shell.execute_reply.started":"2021-06-22T02:44:45.111271Z","shell.execute_reply":"2021-06-22T02:44:45.121251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\n\n# pbar = tqdm(total=len(train))\nfor i, id, dataset_label in train[['Id', 'ext_cleaned_label2']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n#     labels = dataset_label.split('|')\n#     labels = [clean_training_text(label) for label in labels]\n    labels = dataset_label\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence for word in ['data', 'study', 'from']): \n            if np.random.rand(1)[0] > 0.6:\n                ner_data.append(tags)\n                cnt_neg += 1\n    \n    # process bar\n#     pbar.update(1)\n#     pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:44:46.598802Z","iopub.execute_input":"2021-06-22T02:44:46.599135Z","iopub.status.idle":"2021-06-22T02:47:59.038033Z","shell.execute_reply.started":"2021-06-22T02:44:46.599105Z","shell.execute_reply":"2021-06-22T02:47:59.037156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T02:47:59.039511Z","iopub.execute_input":"2021-06-22T02:47:59.039840Z","iopub.status.idle":"2021-06-22T02:47:59.046149Z","shell.execute_reply.started":"2021-06-22T02:47:59.039804Z","shell.execute_reply":"2021-06-22T02:47:59.045161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in ner_data: \n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T22:48:52.151313Z","iopub.execute_input":"2021-06-21T22:48:52.151640Z","iopub.status.idle":"2021-06-21T22:48:55.301120Z","shell.execute_reply.started":"2021-06-21T22:48:52.151608Z","shell.execute_reply":"2021-06-21T22:48:55.300256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'xlm-roberta-base' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T22:48:58.140251Z","iopub.execute_input":"2021-06-21T22:48:58.140584Z","iopub.status.idle":"2021-06-21T22:49:18.481764Z","shell.execute_reply.started":"2021-06-21T22:48:58.140553Z","shell.execute_reply":"2021-06-21T22:49:18.480855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}