{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:20.779656Z","iopub.execute_input":"2023-07-14T13:48:20.780992Z","iopub.status.idle":"2023-07-14T13:48:20.793047Z","shell.execute_reply.started":"2023-07-14T13:48:20.780957Z","shell.execute_reply":"2023-07-14T13:48:20.792059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_SAMPLE = None\ntrain_path = '/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\npaper_train_folder = '/kaggle/input/coleridgeinitiative-show-us-the-data/train'","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:20.794995Z","iopub.execute_input":"2023-07-14T13:48:20.795439Z","iopub.status.idle":"2023-07-14T13:48:20.961496Z","shell.execute_reply.started":"2023-07-14T13:48:20.795407Z","shell.execute_reply":"2023-07-14T13:48:20.960513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cleaned_label'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:20.963165Z","iopub.execute_input":"2023-07-14T13:48:20.963835Z","iopub.status.idle":"2023-07-14T13:48:20.977904Z","shell.execute_reply.started":"2023-07-14T13:48:20.963802Z","shell.execute_reply":"2023-07-14T13:48:20.976804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_data = train['cleaned_label']\nlabel_counts = column_data.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:20.980992Z","iopub.execute_input":"2023-07-14T13:48:20.981673Z","iopub.status.idle":"2023-07-14T13:48:20.992392Z","shell.execute_reply.started":"2023-07-14T13:48:20.981608Z","shell.execute_reply":"2023-07-14T13:48:20.991470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 8))  # Set the size of the figure\nlabel_counts.plot(kind='bar')\nplt.xlabel('Labels')\nplt.ylabel('Frequency')\nplt.title('Histogram of Labels')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:20.993841Z","iopub.execute_input":"2023-07-14T13:48:20.994524Z","iopub.status.idle":"2023-07-14T13:48:24.629782Z","shell.execute_reply.started":"2023-07-14T13:48:20.994494Z","shell.execute_reply":"2023-07-14T13:48:24.628805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_counts = train['cleaned_label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.631205Z","iopub.execute_input":"2023-07-14T13:48:24.631867Z","iopub.status.idle":"2023-07-14T13:48:24.644710Z","shell.execute_reply.started":"2023-07-14T13:48:24.631831Z","shell.execute_reply":"2023-07-14T13:48:24.643858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_counts","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.646034Z","iopub.execute_input":"2023-07-14T13:48:24.646660Z","iopub.status.idle":"2023-07-14T13:48:24.669373Z","shell.execute_reply.started":"2023-07-14T13:48:24.646618Z","shell.execute_reply":"2023-07-14T13:48:24.668207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_labels = label_counts[label_counts >= 500].index","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.671066Z","iopub.execute_input":"2023-07-14T13:48:24.671468Z","iopub.status.idle":"2023-07-14T13:48:24.679795Z","shell.execute_reply.started":"2023-07-14T13:48:24.671437Z","shell.execute_reply":"2023-07-14T13:48:24.678659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(valid_labels)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.681296Z","iopub.execute_input":"2023-07-14T13:48:24.681784Z","iopub.status.idle":"2023-07-14T13:48:24.700415Z","shell.execute_reply.started":"2023-07-14T13:48:24.681754Z","shell.execute_reply":"2023-07-14T13:48:24.699091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_train = train[train['cleaned_label'].isin(valid_labels)]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.706146Z","iopub.execute_input":"2023-07-14T13:48:24.707162Z","iopub.status.idle":"2023-07-14T13:48:24.726115Z","shell.execute_reply.started":"2023-07-14T13:48:24.707125Z","shell.execute_reply":"2023-07-14T13:48:24.725037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(filtered_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.727776Z","iopub.execute_input":"2023-07-14T13:48:24.728205Z","iopub.status.idle":"2023-07-14T13:48:24.736617Z","shell.execute_reply.started":"2023-07-14T13:48:24.728139Z","shell.execute_reply":"2023-07-14T13:48:24.735574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n\n\n# Split the filtered dataset into training and testing sets\ntrain_df, test_df = train_test_split(filtered_train, test_size=0.25, random_state=42)\n\n# Save the training and testing sets to CSV\ntrain_df.to_csv('filtered_train_train.csv', index=False)\ntest_df.to_csv('filtered_train_test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:24.738291Z","iopub.execute_input":"2023-07-14T13:48:24.739328Z","iopub.status.idle":"2023-07-14T13:48:26.069441Z","shell.execute_reply.started":"2023-07-14T13:48:24.739295Z","shell.execute_reply":"2023-07-14T13:48:26.068294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:26.074828Z","iopub.execute_input":"2023-07-14T13:48:26.077388Z","iopub.status.idle":"2023-07-14T13:48:26.088951Z","shell.execute_reply.started":"2023-07-14T13:48:26.077349Z","shell.execute_reply":"2023-07-14T13:48:26.087782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train_df)}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:26.093372Z","iopub.execute_input":"2023-07-14T13:48:26.093953Z","iopub.status.idle":"2023-07-14T13:48:26.946488Z","shell.execute_reply.started":"2023-07-14T13:48:26.093919Z","shell.execute_reply":"2023-07-14T13:48:26.945417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in train_df['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:48:26.948077Z","iopub.execute_input":"2023-07-14T13:48:26.948435Z","iopub.status.idle":"2023-07-14T13:49:01.873552Z","shell.execute_reply.started":"2023-07-14T13:48:26.948402Z","shell.execute_reply":"2023-07-14T13:49:01.872534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(test_df)}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:01.876984Z","iopub.execute_input":"2023-07-14T13:49:01.877322Z","iopub.status.idle":"2023-07-14T13:49:02.112248Z","shell.execute_reply.started":"2023-07-14T13:49:01.877295Z","shell.execute_reply":"2023-07-14T13:49:02.111294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Create a dictionary with the required columns\nsubmission_data = {\n    'Id': test_df['Id'],\n    'PredictionString': [None] * len(test_df)\n}\n\n# Create a DataFrame from the dictionary\nsample_submission = pd.DataFrame(submission_data)\n\n# Save the DataFrame as sample_submission.csv\nsample_submission.to_csv('sample_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:02.114091Z","iopub.execute_input":"2023-07-14T13:49:02.114717Z","iopub.status.idle":"2023-07-14T13:49:02.133096Z","shell.execute_reply.started":"2023-07-14T13:49:02.114684Z","shell.execute_reply":"2023-07-14T13:49:02.132179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:02.134395Z","iopub.execute_input":"2023-07-14T13:49:02.135673Z","iopub.status.idle":"2023-07-14T13:49:02.148311Z","shell.execute_reply.started":"2023-07-14T13:49:02.135635Z","shell.execute_reply":"2023-07-14T13:49:02.147184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:02.149877Z","iopub.execute_input":"2023-07-14T13:49:02.150417Z","iopub.status.idle":"2023-07-14T13:49:02.156554Z","shell.execute_reply.started":"2023-07-14T13:49:02.150386Z","shell.execute_reply":"2023-07-14T13:49:02.155660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_test_folder = '/kaggle/input/coleridgeinitiative-show-us-the-data/train'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper_test = json.load(f)\n        papers[paper_id] = paper_test","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:02.157704Z","iopub.execute_input":"2023-07-14T13:49:02.158617Z","iopub.status.idle":"2023-07-14T13:49:11.220714Z","shell.execute_reply.started":"2023-07-14T13:49:02.158585Z","shell.execute_reply":"2023-07-14T13:49:11.219698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(papers)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:11.222445Z","iopub.execute_input":"2023-07-14T13:49:11.222840Z","iopub.status.idle":"2023-07-14T13:49:11.229539Z","shell.execute_reply.started":"2023-07-14T13:49:11.222804Z","shell.execute_reply":"2023-07-14T13:49:11.228509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install /kaggle/input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install /kaggle/input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install /kaggle/input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:11.231341Z","iopub.execute_input":"2023-07-14T13:49:11.232137Z","iopub.status.idle":"2023-07-14T13:49:55.295167Z","shell.execute_reply.started":"2023-07-14T13:49:11.232105Z","shell.execute_reply":"2023-07-14T13:49:55.293959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.297038Z","iopub.execute_input":"2023-07-14T13:49:55.297671Z","iopub.status.idle":"2023-07-14T13:49:55.592859Z","shell.execute_reply.started":"2023-07-14T13:49:55.297631Z","shell.execute_reply":"2023-07-14T13:49:55.591897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64 \nOVERLAP = 20\nPREDICT_BATCH = 64000 ","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.594273Z","iopub.execute_input":"2023-07-14T13:49:55.594620Z","iopub.status.idle":"2023-07-14T13:49:55.600124Z","shell.execute_reply.started":"2023-07-14T13:49:55.594588Z","shell.execute_reply":"2023-07-14T13:49:55.599163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train_df[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.601616Z","iopub.execute_input":"2023-07-14T13:49:55.602681Z","iopub.status.idle":"2023-07-14T13:49:55.636233Z","shell.execute_reply.started":"2023-07-14T13:49:55.602648Z","shell.execute_reply":"2023-07-14T13:49:55.635271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.637446Z","iopub.execute_input":"2023-07-14T13:49:55.637791Z","iopub.status.idle":"2023-07-14T13:49:55.647459Z","shell.execute_reply.started":"2023-07-14T13:49:55.637757Z","shell.execute_reply":"2023-07-14T13:49:55.646476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.648730Z","iopub.execute_input":"2023-07-14T13:49:55.649317Z","iopub.status.idle":"2023-07-14T13:49:55.655819Z","shell.execute_reply.started":"2023-07-14T13:49:55.649285Z","shell.execute_reply":"2023-07-14T13:49:55.654882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:49:55.662004Z","iopub.execute_input":"2023-07-14T13:49:55.664057Z","iopub.status.idle":"2023-07-14T13:50:19.354929Z","shell.execute_reply.started":"2023-07-14T13:49:55.664024Z","shell.execute_reply":"2023-07-14T13:50:19.353875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:50:19.356282Z","iopub.execute_input":"2023-07-14T13:50:19.357105Z","iopub.status.idle":"2023-07-14T13:50:19.365157Z","shell.execute_reply.started":"2023-07-14T13:50:19.357069Z","shell.execute_reply":"2023-07-14T13:50:19.364150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:50:19.366716Z","iopub.execute_input":"2023-07-14T13:50:19.367303Z","iopub.status.idle":"2023-07-14T13:50:20.341916Z","shell.execute_reply.started":"2023-07-14T13:50:19.367270Z","shell.execute_reply":"2023-07-14T13:50:20.340592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:50:20.343837Z","iopub.execute_input":"2023-07-14T13:50:20.345027Z","iopub.status.idle":"2023-07-14T13:50:20.358091Z","shell.execute_reply.started":"2023-07-14T13:50:20.344989Z","shell.execute_reply":"2023-07-14T13:50:20.356967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\n\npbar = tqdm(total=len(train_df))\nfor i, id, dataset_label in train_df[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence.lower() for word in ['data', 'study']): \n            ner_data.append(tags)\n            cnt_neg += 1\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:50:20.359938Z","iopub.execute_input":"2023-07-14T13:50:20.360285Z","iopub.status.idle":"2023-07-14T13:51:42.886087Z","shell.execute_reply.started":"2023-07-14T13:50:20.360255Z","shell.execute_reply":"2023-07-14T13:51:42.885020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:51:42.887783Z","iopub.execute_input":"2023-07-14T13:51:42.888342Z","iopub.status.idle":"2023-07-14T13:51:59.371478Z","shell.execute_reply.started":"2023-07-14T13:51:42.888307Z","shell.execute_reply":"2023-07-14T13:51:59.370490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python /kaggle/input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'bert-base-cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train ","metadata":{"execution":{"iopub.status.busy":"2023-07-14T13:51:59.373143Z","iopub.execute_input":"2023-07-14T13:51:59.373513Z","iopub.status.idle":"2023-07-14T14:38:37.374374Z","shell.execute_reply.started":"2023-07-14T13:51:59.373480Z","shell.execute_reply":"2023-07-14T14:38:37.373167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python /kaggle/input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'bert-base-cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_eval","metadata":{"execution":{"iopub.status.busy":"2023-07-14T14:54:14.493278Z","iopub.execute_input":"2023-07-14T14:54:14.494130Z","iopub.status.idle":"2023-07-14T15:14:01.955571Z","shell.execute_reply.started":"2023-07-14T14:54:14.494094Z","shell.execute_reply":"2023-07-14T15:14:01.954225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = './output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = 'train_ner.json'\nVAL_PATH = 'train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2023-07-14T14:41:58.393273Z","iopub.execute_input":"2023-07-14T14:41:58.393640Z","iopub.status.idle":"2023-07-14T14:41:58.400026Z","shell.execute_reply.started":"2023-07-14T14:41:58.393611Z","shell.execute_reply":"2023-07-14T14:41:58.398765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T14:42:16.171386Z","iopub.execute_input":"2023-07-14T14:42:16.171791Z","iopub.status.idle":"2023-07-14T14:42:33.313402Z","shell.execute_reply.started":"2023-07-14T14:42:16.171736Z","shell.execute_reply":"2023-07-14T14:42:33.312292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2023-07-14T14:42:33.315523Z","iopub.execute_input":"2023-07-14T14:42:33.316257Z","iopub.status.idle":"2023-07-14T14:42:33.322408Z","shell.execute_reply.started":"2023-07-14T14:42:33.316214Z","shell.execute_reply":"2023-07-14T14:42:33.321337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T14:44:34.247831Z","iopub.execute_input":"2023-07-14T14:44:34.248228Z","iopub.status.idle":"2023-07-14T14:44:34.253855Z","shell.execute_reply.started":"2023-07-14T14:44:34.248199Z","shell.execute_reply":"2023-07-14T14:44:34.252914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python /kaggle/input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:14:01.958411Z","iopub.execute_input":"2023-07-14T15:14:01.958885Z","iopub.status.idle":"2023-07-14T15:14:01.965934Z","shell.execute_reply.started":"2023-07-14T15:14:01.958844Z","shell.execute_reply":"2023-07-14T15:14:01.964783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade transformers\n!pip install --upgrade accelerate","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:14:01.967277Z","iopub.execute_input":"2023-07-14T15:14:01.967735Z","iopub.status.idle":"2023-07-14T15:14:26.501011Z","shell.execute_reply.started":"2023-07-14T15:14:01.967700Z","shell.execute_reply":"2023-07-14T15:14:26.499804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:14:26.505589Z","iopub.execute_input":"2023-07-14T15:14:26.505937Z","iopub.status.idle":"2023-07-14T15:22:59.882583Z","shell.execute_reply.started":"2023-07-14T15:14:26.505907Z","shell.execute_reply":"2023-07-14T15:22:59.881341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:22:59.886521Z","iopub.execute_input":"2023-07-14T15:22:59.886892Z","iopub.status.idle":"2023-07-14T15:23:00.052081Z","shell.execute_reply.started":"2023-07-14T15:22:59.886863Z","shell.execute_reply":"2023-07-14T15:23:00.051049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:00.053864Z","iopub.execute_input":"2023-07-14T15:23:00.054387Z","iopub.status.idle":"2023-07-14T15:23:01.049579Z","shell.execute_reply.started":"2023-07-14T15:23:00.054345Z","shell.execute_reply":"2023-07-14T15:23:01.048416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:01.053666Z","iopub.execute_input":"2023-07-14T15:23:01.055009Z","iopub.status.idle":"2023-07-14T15:23:01.063674Z","shell.execute_reply.started":"2023-07-14T15:23:01.054972Z","shell.execute_reply":"2023-07-14T15:23:01.062481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:01.065253Z","iopub.execute_input":"2023-07-14T15:23:01.065788Z","iopub.status.idle":"2023-07-14T15:23:01.100946Z","shell.execute_reply.started":"2023-07-14T15:23:01.065737Z","shell.execute_reply":"2023-07-14T15:23:01.099724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:01.102826Z","iopub.execute_input":"2023-07-14T15:23:01.103190Z","iopub.status.idle":"2023-07-14T15:23:01.110703Z","shell.execute_reply.started":"2023-07-14T15:23:01.103157Z","shell.execute_reply":"2023-07-14T15:23:01.109630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\nfor literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n    if literal_match:\n        final_predictions.append(literal_match)\n    else:\n        final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:01.114530Z","iopub.execute_input":"2023-07-14T15:23:01.114885Z","iopub.status.idle":"2023-07-14T15:23:01.123540Z","shell.execute_reply.started":"2023-07-14T15:23:01.114859Z","shell.execute_reply":"2023-07-14T15:23:01.122523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:23:01.124893Z","iopub.execute_input":"2023-07-14T15:23:01.125420Z","iopub.status.idle":"2023-07-14T15:23:01.143715Z","shell.execute_reply.started":"2023-07-14T15:23:01.125386Z","shell.execute_reply":"2023-07-14T15:23:01.142730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T15:44:53.437913Z","iopub.execute_input":"2023-07-14T15:44:53.438304Z","iopub.status.idle":"2023-07-14T15:44:53.450216Z","shell.execute_reply.started":"2023-07-14T15:44:53.438276Z","shell.execute_reply":"2023-07-14T15:44:53.449144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}