{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"# !git clone https://github.com/pytorch/fairseq\n# %cd fairseq\n# !pip install --editable ./\n# %cd ..\n\n# !pip install pytorch-transformers\n\n# import os\n# PARAM_SET='base' # change to large to use the large architecture\n# os.environ[\"PARAM_SET\"] = f\"{PARAM_SET}\"\n\n# # clone the repo\n# !git clone https://github.com/mohammadKhalifa/xlm-roberta-ner.git\n# %cd xlm-roberta-ner\n# !mkdir pretrained_models \n# !wget -P pretrained_models https://dl.fbaipublicfiles.com/fairseq/models/xlmr.$PARAM_SET.tar.gz\n# !tar xzvf pretrained_models/xlmr.$PARAM_SET.tar.gz  --directory pretrained_models/\n# !rm -r pretrained_models/xlmr.$PARAM_SET.tar.gz\n# %cd ..","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:06:20.493843Z","iopub.execute_input":"2021-06-13T12:06:20.494255Z","iopub.status.idle":"2021-06-13T12:06:20.498801Z","shell.execute_reply.started":"2021-06-13T12:06:20.494161Z","shell.execute_reply":"2021-06-13T12:06:20.498007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LENGTH = 1\nFL_TH = 0.75\n\nMAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:06:20.890634Z","iopub.execute_input":"2021-06-13T12:06:20.891001Z","iopub.status.idle":"2021-06-13T12:06:20.896048Z","shell.execute_reply.started":"2021-06-13T12:06:20.890966Z","shell.execute_reply":"2021-06-13T12:06:20.894830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\n# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.display import clear_output\n\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:06:21.466971Z","iopub.execute_input":"2021-06-13T12:06:21.467345Z","iopub.status.idle":"2021-06-13T12:06:55.569143Z","shell.execute_reply.started":"2021-06-13T12:06:21.467315Z","shell.execute_reply":"2021-06-13T12:06:55.567872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://huggingface.co/transformers/_modules/transformers/trainer_utils.html\ndef set_seed(seed: int):\n    \"\"\"\n    Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch`` and/or ``tf`` (if\n    installed).\n\n    Args:\n        seed (:obj:`int`): The seed to set.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    # ^^ safe to call this function even if cuda is not available\n    \n    print(f'Setted Pipeline SEED = {SEED}')\n\n\nset_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:06:55.571408Z","iopub.execute_input":"2021-06-13T12:06:55.571751Z","iopub.status.idle":"2021-06-13T12:06:55.580564Z","shell.execute_reply.started":"2021-06-13T12:06:55.571719Z","shell.execute_reply":"2021-06-13T12:06:55.578358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Internal Training Data","metadata":{}},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\n\ndef jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\n\ndef find_data_sets_id(publication_id: int) -> list:\n    data_set_ids = []\n    for class_ in data_set_citations:\n        if class_['publication_id'] == publication_id:\n            data_set_ids.append(class_['data_set_id'])\n    return data_set_ids\n\n\ndef find_data_set_citations_mention_list(publication_id: int) -> str:\n    mention_list = []\n    for class_ in data_set_citations:\n        if class_['publication_id'] == publication_id:\n            mention_list.append( '|'.join(class_['mention_list']) )\n    return '|'.join( [label for label in mention_list if label != ''] )\n\n\ndef find_data_sets_title(data_set_id: int) -> str:\n    for class_ in data_sets:\n        if class_['data_set_id'] == data_set_id:\n            return class_['title']\n        \n\ndef RichContextDF(publications: '.json') -> pd.DataFrame:\n    publication_id = []\n    text_file_name = []\n    citations_mention_list = []\n    data_sets_title = []\n    for class_ in publications:\n        publication_id.append(class_['publication_id']) # to get data_set_citations\n        text_file_name.append(class_['text_file_name']) # to get text_file\n        \n        # to get citations_mention_list\n        citations_mention_list.append( find_data_set_citations_mention_list(class_['publication_id']) )\n        \n        # to get data_sets_title\n        data_sets_title_temp = []\n        for data_sets_id in find_data_sets_id( class_['publication_id'] ):\n            data_sets_title_temp.append( find_data_sets_title(data_sets_id) )\n        data_sets_title.append('||'.join(data_sets_title_temp))\n    \n    return pd.DataFrame({\n        'publication_id': publication_id,\n        'text_file_name': text_file_name,\n        'citations_mention_list': citations_mention_list,\n        'data_sets_title': data_sets_title\n    })","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:06:55.582389Z","iopub.execute_input":"2021-06-13T12:06:55.582830Z","iopub.status.idle":"2021-06-13T12:06:55.600496Z","shell.execute_reply.started":"2021-06-13T12:06:55.582777Z","shell.execute_reply":"2021-06-13T12:06:55.599411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridge-training-set/colerdige_train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\nprint('train size before agg.:', len(train))\n\ntrain = train[:MAX_SAMPLE]\n# Group by publication, training labels should have the same form as expected output.\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()    \nprint('train size after agg.:', len(train))\n\ntrain = train.sort_values(by=['Id'])\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:08:56.864021Z","iopub.execute_input":"2021-06-13T12:08:56.864411Z","iopub.status.idle":"2021-06-13T12:08:57.626467Z","shell.execute_reply.started":"2021-06-13T12:08:56.864371Z","shell.execute_reply":"2021-06-13T12:08:57.625142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pseudo","metadata":{}},{"cell_type":"code","source":"# pseudo_train_path = '../input/coleridge-pseudolabelsv2-0585/submission.csv'\n\n# pseudo_train = pd.read_csv(pseudo_train_path)\n# print('pseudo_train size before drop_duplicates.:', len(pseudo_train))\n\n# pseudo_train = pseudo_train[:MAX_SAMPLE]\n# pseudo_train = pseudo_train.drop_duplicates(subset='Id')\n# print('pseudo_train size after drop_duplicates:', len(pseudo_train), '\\n')\n\n# pseudo_train = pseudo_train.sort_values(by=['Id']).reset_index(drop=True)\n# pseudo_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:08:59.073745Z","iopub.execute_input":"2021-06-13T12:08:59.074161Z","iopub.status.idle":"2021-06-13T12:08:59.078818Z","shell.execute_reply.started":"2021-06-13T12:08:59.074128Z","shell.execute_reply":"2021-06-13T12:08:59.077914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train + Pseudo","metadata":{}},{"cell_type":"code","source":"# train['dataset_label'] = train['dataset_label'] + '|' + pseudo_train['PredictionString']\n\n# del pseudo_train\n\n# train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:00.172910Z","iopub.execute_input":"2021-06-13T12:09:00.173329Z","iopub.status.idle":"2021-06-13T12:09:00.178737Z","shell.execute_reply.started":"2021-06-13T12:09:00.173289Z","shell.execute_reply":"2021-06-13T12:09:00.177277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_temp = []\n# for labels in tqdm(train['dataset_label']):\n#     labels = labels.split('|')\n#     filtered_labels = []\n#     for label in labels:\n#         if len(filtered_labels) == 0 or all(jaccard_similarity(text_cleaning(label), text_cleaning(got_label)) < FL_TH for got_label in filtered_labels):\n#             filtered_labels.append(label)\n#     train_temp.append('|'.join(filtered_labels))\n# train['dataset_label'] = train_temp\n# train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:11.083129Z","iopub.execute_input":"2021-06-13T12:09:11.083526Z","iopub.status.idle":"2021-06-13T12:09:11.087861Z","shell.execute_reply.started":"2021-06-13T12:09:11.083492Z","shell.execute_reply":"2021-06-13T12:09:11.086679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# External Training Data","metadata":{}},{"cell_type":"code","source":"with open(f'../input/rich-context-competition-train-testtargz/train_test/publications.json', 'r') as f:\n    publications = json.load(f)\nwith open(f'../input/rich-context-competition-train-testtargz/train_test/data_set_citations.json', 'r') as f:\n    data_set_citations = json.load(f)\nwith open(f'../input/rich-context-competition-train-testtargz/train_test/data_sets.json', 'r') as f:\n    data_sets = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:17.714335Z","iopub.execute_input":"2021-06-13T12:09:17.714740Z","iopub.status.idle":"2021-06-13T12:09:18.603214Z","shell.execute_reply.started":"2021-06-13T12:09:17.714707Z","shell.execute_reply":"2021-06-13T12:09:18.602220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RichContext_train = RichContextDF(publications)\nRichContext_train = RichContext_train[ RichContext_train['citations_mention_list'] != '' ]\nRichContext_train","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:18.605113Z","iopub.execute_input":"2021-06-13T12:09:18.605528Z","iopub.status.idle":"2021-06-13T12:09:23.698179Z","shell.execute_reply.started":"2021-06-13T12:09:18.605482Z","shell.execute_reply":"2021-06-13T12:09:23.697065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to NER format","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:31.396170Z","iopub.execute_input":"2021-06-13T12:09:31.396525Z","iopub.status.idle":"2021-06-13T12:09:31.409134Z","shell.execute_reply.started":"2021-06-13T12:09:31.396496Z","shell.execute_reply":"2021-06-13T12:09:31.408144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\n\npbar = tqdm(total = len(RichContext_train))\nfor paper_id, dataset_labels in RichContext_train[['publication_id', 'citations_mention_list']].itertuples(index=False):\n    # labels\n    labels = dataset_labels.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # paper\n    with open(f'../input/rich-context-competition-train-testtargz/train_test/files/text/{paper_id}.txt', 'r') as f:\n        paper = f.readlines()\n    paper = [line[:-1] for line in paper]\n    content = ' '.join(paper)\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for sentence in content.split('.')])\n    sentences = shorten_sentences(sentences)\n    sentences = [sentence for sentence in sentences if len(sentence) > LENGTH]\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence.lower() for word in ['data', 'study']): \n            ner_data.append(tags)\n            cnt_neg += 1\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\npbar.close()\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:09:32.039520Z","iopub.execute_input":"2021-06-13T12:09:32.039879Z","iopub.status.idle":"2021-06-13T12:11:09.137152Z","shell.execute_reply.started":"2021-06-13T12:09:32.039849Z","shell.execute_reply":"2021-06-13T12:11:09.135841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(train['Id'].unique()):\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:11:09.139216Z","iopub.execute_input":"2021-06-13T12:11:09.139547Z","iopub.status.idle":"2021-06-13T12:12:14.746665Z","shell.execute_reply.started":"2021-06-13T12:11:09.139516Z","shell.execute_reply":"2021-06-13T12:12:14.744995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pbar = tqdm(total=len(train))\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > LENGTH] # only accept sentences with length > 10 chars\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence.lower() for word in ['data', 'study']): \n            ner_data.append(tags)\n            cnt_neg += 1\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\npbar.close()\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:12:14.748611Z","iopub.execute_input":"2021-06-13T12:12:14.749055Z","iopub.status.idle":"2021-06-13T12:15:28.919624Z","shell.execute_reply.started":"2021-06-13T12:12:14.749006Z","shell.execute_reply":"2021-06-13T12:15:28.918286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving Training Data","metadata":{}},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in tqdm(ner_data):\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:15:28.921092Z","iopub.execute_input":"2021-06-13T12:15:28.921393Z","iopub.status.idle":"2021-06-13T12:16:12.530085Z","shell.execute_reply.started":"2021-06-13T12:15:28.921364Z","shell.execute_reply":"2021-06-13T12:16:12.528730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ner_data[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.533881Z","iopub.execute_input":"2021-06-13T12:16:12.534340Z","iopub.status.idle":"2021-06-13T12:16:12.539567Z","shell.execute_reply.started":"2021-06-13T12:16:12.534291Z","shell.execute_reply":"2021-06-13T12:16:12.538375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nes_check = []\n# for row in ner_data:\n#     words, nes = list(zip(*row))\n#     nes_check.append(nes)\n\n# flat_list = []\n# for sublist in nes_check:\n#     for item in sublist:\n#         flat_list.append(item)\n# set(flat_list)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.540833Z","iopub.execute_input":"2021-06-13T12:16:12.541115Z","iopub.status.idle":"2021-06-13T12:16:12.550813Z","shell.execute_reply.started":"2021-06-13T12:16:12.541087Z","shell.execute_reply":"2021-06-13T12:16:12.549988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune a XLM-RoBERTa model for NER","metadata":{}},{"cell_type":"code","source":"# !python ./xlm-roberta-ner/main.py \\\n#       --data_dir=data/coNLL-2003/ \\\n#       --task_name=ner \\\n#       --output_dir=model_dir/ \\\n#       --max_seq_length=16 \\\n#       --num_train_epochs 1 \\\n#       --do_eval \\\n#       --warmup_proportion=0.1 \\\n#       --pretrained_path pretrained_models/xlmr.$PARAM_SET/ \\\n#       --learning_rate 0.00007 \\\n#       --do_train \\\n#       --eval_on test \\\n#       --train_batch_size 4 \\\n#       -- dropout 0.2","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.552001Z","iopub.execute_input":"2021-06-13T12:16:12.552408Z","iopub.status.idle":"2021-06-13T12:16:12.562638Z","shell.execute_reply.started":"2021-06-13T12:16:12.552369Z","shell.execute_reply":"2021-06-13T12:16:12.561808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from transformers import RobertaTokenizerFast\n# tokenizer = RobertaTokenizerFast.from_pretrained(\"roberta-base\", add_prefix_space=True)\n# tokenizer.save_pretrained('./RobertaTokenizerFastt')\n# # !mv ./RobertaTokenizerFast/tokenizer_config.json ./RobertaTokenizerFast/config.json","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.563855Z","iopub.execute_input":"2021-06-13T12:16:12.564269Z","iopub.status.idle":"2021-06-13T12:16:12.580875Z","shell.execute_reply.started":"2021-06-13T12:16:12.564227Z","shell.execute_reply":"2021-06-13T12:16:12.579329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !git clone https://github.com/huggingface/transformers/examples/pytorch/token-classification","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.582643Z","iopub.execute_input":"2021-06-13T12:16:12.583005Z","iopub.status.idle":"2021-06-13T12:16:12.595133Z","shell.execute_reply.started":"2021-06-13T12:16:12.582954Z","shell.execute_reply":"2021-06-13T12:16:12.594080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip uninstall transformers --yes\n# !pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.596358Z","iopub.execute_input":"2021-06-13T12:16:12.596949Z","iopub.status.idle":"2021-06-13T12:16:12.611026Z","shell.execute_reply.started":"2021-06-13T12:16:12.596913Z","shell.execute_reply":"2021-06-13T12:16:12.609455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/coleridge-ner-robertabase/kaggle_run_ner_roberta-base.py \\\n--model_name_or_path 'roberta-base' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 42 \\\n--do_train ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T12:16:12.612464Z","iopub.execute_input":"2021-06-13T12:16:12.612750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/eriknovak/pytorch-roberta-named-entity-recognition\n\n# # visualization libraries\n# import matplotlib.pyplot as plt\n# import numpy as np\n\n# # pytorch libraries\n# import torch # the main pytorch library\n# import torch.nn as nn # the sub-library containing Softmax, Module and other useful functions\n# import torch.optim as optim # the sub-library containing the common optimizers (SGD, Adam, etc.)\n\n# # huggingface's transformers library\n# from transformers import RobertaForTokenClassification, RobertaTokenizer\n\n# # huggingface's datasets library\n# from datasets import load_dataset\n\n# # the tqdm library used to show the iteration progress\n# import tqdm\n# tqdmn = tqdm.notebook.tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-13T08:40:27.709868Z","iopub.execute_input":"2021-06-13T08:40:27.710245Z","iopub.status.idle":"2021-06-13T08:40:29.833280Z","shell.execute_reply.started":"2021-06-13T08:40:27.710214Z","shell.execute_reply":"2021-06-13T08:40:29.832463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# roberta_version = 'roberta-base'\n# tokenizer = RobertaTokenizer.from_pretrained(roberta_version)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T08:40:35.987319Z","iopub.execute_input":"2021-06-13T08:40:35.987991Z","iopub.status.idle":"2021-06-13T08:40:39.929080Z","shell.execute_reply.started":"2021-06-13T08:40:35.987925Z","shell.execute_reply":"2021-06-13T08:40:39.928164Z"},"trusted":true},"execution_count":null,"outputs":[]}]}