{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports & Preamble","metadata":{}},{"cell_type":"code","source":"!pip install -qU --no-warn-conflicts transformers --no-index --find-links=file:///kaggle/input/coleridge-packages\n!pip install -qU --no-warn-conflicts tokenizers --no-index --find-links=file:///kaggle/input/coleridge-packages\n!pip install -qU --no-warn-conflicts datasets --no-index --find-links=file:///kaggle/input/coleridge-packages\n!pip install -qU --no-warn-conflicts fsspec --no-index --find-links=file:///kaggle/input/coleridge-packages","metadata":{"execution":{"iopub.status.busy":"2021-07-03T10:20:26.803318Z","iopub.execute_input":"2021-07-03T10:20:26.803966Z","iopub.status.idle":"2021-07-03T10:20:38.663116Z","shell.execute_reply.started":"2021-07-03T10:20:26.803846Z","shell.execute_reply":"2021-07-03T10:20:38.661953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import annotations\nimport os\nimport pandas as pd\nimport json\nimport re\nfrom typing import Iterable\nfrom tqdm.notebook import tqdm\n\nfrom transformers import (\n    BigBirdTokenizerFast,\n)\nfrom datasets import (\n    Dataset,\n    Features,\n    Sequence,\n    Value,\n    ClassLabel,\n)\n\nfrom coleridge_helpers import (\n    clean_text,\n    get_text_as_word_array,\n    find_last_period_in_string_array,\n    get_tags_for_snippet,\n    get_snippets_from_paper,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T10:28:07.487399Z","iopub.execute_input":"2021-07-03T10:28:07.487716Z","iopub.status.idle":"2021-07-03T10:28:09.422957Z","shell.execute_reply.started":"2021-07-03T10:28:07.487685Z","shell.execute_reply":"2021-07-03T10:28:09.421913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load & Preprocess Data","metadata":{}},{"cell_type":"code","source":"dataset_path = \"../input/coleridgeinitiative-show-us-the-data/\"\ntrainfiles_path = dataset_path + \"train/\"\ntrain_metadata = pd.read_csv(dataset_path + \"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-03T10:28:12.649887Z","iopub.execute_input":"2021-07-03T10:28:12.650248Z","iopub.status.idle":"2021-07-03T10:28:12.785357Z","shell.execute_reply.started":"2021-07-03T10:28:12.650214Z","shell.execute_reply":"2021-07-03T10:28:12.784572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Context Snippets","metadata":{}},{"cell_type":"code","source":"all_snippets = []\n\n# Loop through all the files and create a collection of snippets (training examples). \n# These snippets should provide as much context as possible. So we make them as close \n# as we can to the maximum length BigBird will accept while keeping sections intact.\n\nfor filename in tqdm(os.listdir(trainfiles_path)):\n    snippets = get_snippets_from_paper(f\"{trainfiles_path}{filename}\")\n    all_snippets.extend(snippets)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-03T10:31:46.936745Z","iopub.execute_input":"2021-07-03T10:31:46.937065Z","iopub.status.idle":"2021-07-03T10:33:49.185047Z","shell.execute_reply.started":"2021-07-03T10:31:46.937034Z","shell.execute_reply":"2021-07-03T10:33:49.184129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Find Labels in Snippets and Create Corresponding Tags","metadata":{}},{"cell_type":"code","source":"# Create token and tag arrays for each snippet in a dataframe\n\n# N.B. the tokens here are words and punctuation, not the subword tokens \n# that will later be created by the BigBird Tokenizer\n\n# We need to reduce the size of our dataset in order for a single \n# training epoch to finish within Kaggle's 9hr limit\nFRAC_OF_NEGATIVE_EXAMPLES_TO_KEEP = 0.05\n\nunique_labels = train_metadata[\"dataset_label\"].unique()\nrows = []\nrows_for_snippets_without_datasets = []\n\nfor snippet in tqdm(all_snippets):\n\n    tokens = get_text_as_word_array(snippet)\n    found_labels = set()\n\n    for label in unique_labels:\n        if re.search(f\"\\\\b{label}\\\\b\", snippet):\n            found_labels.add(label)\n\n    tags = get_tags_for_snippet(tokens, found_labels)\n\n    row = {\"tokens\": tokens, \"tags\": tags}\n\n    if len(found_labels) == 0:\n        rows_for_snippets_without_datasets.append(row)\n    else:\n        rows.append(row)\n\nsnippets_negative_samples = pd.DataFrame(rows_for_snippets_without_datasets)\nsnippets_positive_samples = pd.DataFrame(rows)\nsnippets_df = pd.concat(\n    [\n        snippets_positive_samples,\n        snippets_negative_samples.sample(frac=FRAC_OF_NEGATIVE_EXAMPLES_TO_KEEP),\n    ],\n    ignore_index=True,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T10:38:04.576807Z","iopub.execute_input":"2021-07-03T10:38:04.577147Z","iopub.status.idle":"2021-07-03T11:01:02.192063Z","shell.execute_reply.started":"2021-07-03T10:38:04.577114Z","shell.execute_reply":"2021-07-03T11:01:02.190708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a HuggingFace Dataset","metadata":{}},{"cell_type":"code","source":"label2id = {\"O\": 0, \"B\": 1, \"I\": 2}\n\nsnippets_df[\"tags\"] = snippets_df[\"tags\"].apply(\n    lambda tags: [label2id[tag] for tag in tags]\n)\n\nfeatures = Features(\n    {\n        \"tokens\": Sequence(Value(\"string\")),\n        \"tags\": Sequence(ClassLabel(names=[\"O\", \"B\", \"I\"])),\n    }\n)\n\ndataset = Dataset.from_pandas(snippets_df, features=features)\n\ntokenizer = BigBirdTokenizerFast.from_pretrained(\n    \"../input/huggingfacebigbirdrobertabase\"\n)\n\ndef tokenize_and_align_labels(examples):\n    tokenized_inputs = tokenizer(\n        examples[\"tokens\"], truncation=True, is_split_into_words=True\n    )\n\n    labels = []\n    for i, label in enumerate(examples[\"tags\"]):\n        word_ids = tokenized_inputs.word_ids(batch_index=i)\n        previous_word_idx = None\n        label_ids = []\n        for word_idx in word_ids:\n            # Special tokens have a word id that is None. \n            # We set the label to -100 so they are automatically ignored in the loss function.\n            if word_idx is None:\n                label_ids.append(-100)\n            # We set the label for the first token of each word.\n            elif word_idx != previous_word_idx:\n                label_ids.append(label[word_idx])\n            # For the other tokens in a word, we set the label to the current label\n            else:\n                label_ids.append(label[word_idx])\n            previous_word_idx = word_idx\n\n        labels.append(label_ids)\n\n    tokenized_inputs[\"labels\"] = labels\n    return tokenized_inputs\n\n\ntokenized_dataset = dataset.map(tokenize_and_align_labels, batched=True)\ntokenized_dataset = tokenized_dataset.shuffle(seed=42)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T11:05:32.503608Z","iopub.execute_input":"2021-07-03T11:05:32.503975Z","iopub.status.idle":"2021-07-03T11:08:39.472018Z","shell.execute_reply.started":"2021-07-03T11:05:32.503941Z","shell.execute_reply":"2021-07-03T11:08:39.470964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# need to limit the batch size to stop the kernel running out of memory\ntokenized_dataset.to_json(\"tokenized_dataset.json\", batch_size=64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}