{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook runs inference on a fine-tuned Google BigBird model for the Coleridge Intiaitive Show US the data competition.  \n\nIn order to be able to run data-processing, training and inference within Kaggle's 9hr timout limit,  \nI separated out the data preparation, here: https://www.kaggle.com/danieldorosz/show-us-the-data-bigbird-dataprep  \nand the model fine-tuning, here: https://www.kaggle.com/danieldorosz/show-us-the-data-bigbird-fine-tuning  \n\nA chunk of the logic is farmed-out to a coleridge-helpers utility script.   \n\nThe main intuition behind this effort was that I wanted to include as much context as possible in my training examples. \nAlso that I wanted to keep related context together. We have a ready-provided demarkation of context expressed as \nsections in the training data. So what I did was create contextual 'snippets' as my training examples. Each snippet \ncontains one or more sections such that my training examples get as close as possible to BigBird's maximum of 4096\ntokens, without breaking up any sections. If a single section is longer the training example limit, I break it up \nat the last period prior to the limit.  \n\nThe code is very much a rough-and-ready first draft, please don't judge me ;-) There is much to be improved for which \nI didn't have time. This mainly serves as a baseline to assess the score I could expect from this kind of approach.\n\nI ran fine-tuning a couple of times using the last checkpoint from the first (timed-out) run as input to the next.","metadata":{}},{"cell_type":"markdown","source":"# Imports & Preamble","metadata":{}},{"cell_type":"code","source":"!pip install -qU --no-warn-conflicts transformers --no-index --find-links=file:///kaggle/input/coleridge-packages\n!pip install -qU --no-warn-conflicts tokenizers --no-index --find-links=file:///kaggle/input/coleridge-packages","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:39.250423Z","iopub.execute_input":"2021-07-04T18:35:39.250845Z","iopub.status.idle":"2021-07-04T18:35:46.222342Z","shell.execute_reply.started":"2021-07-04T18:35:39.250747Z","shell.execute_reply":"2021-07-04T18:35:46.221296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport torch\nfrom transformers import (\n    BigBirdForTokenClassification,\n    BigBirdConfig,\n    BigBirdTokenizerFast,\n)\n\nfrom coleridge_helpers import (\n    clean_text,\n    get_snippets_from_paper,\n    find_datasets_by_literal_matching,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:46.224114Z","iopub.execute_input":"2021-07-04T18:35:46.224444Z","iopub.status.idle":"2021-07-04T18:35:52.334738Z","shell.execute_reply.started":"2021-07-04T18:35:46.224408Z","shell.execute_reply":"2021-07-04T18:35:52.333941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:52.336489Z","iopub.execute_input":"2021-07-04T18:35:52.336825Z","iopub.status.idle":"2021-07-04T18:35:52.385522Z","shell.execute_reply.started":"2021-07-04T18:35:52.336776Z","shell.execute_reply":"2021-07-04T18:35:52.384533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_path = \"../input/coleridgeinitiative-show-us-the-data/\"\ntestfiles_path = dataset_path + \"test/\"","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:52.387428Z","iopub.execute_input":"2021-07-04T18:35:52.388055Z","iopub.status.idle":"2021-07-04T18:35:52.392742Z","shell.execute_reply.started":"2021-07-04T18:35:52.388017Z","shell.execute_reply":"2021-07-04T18:35:52.391706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a set of dataset titles we will use for literal string matching\n\ntrain_metadata = pd.read_csv(dataset_path + \"train.csv\")\n\ntrain_titles = set(train_metadata[\"dataset_title\"].unique())\ntrain_titles = {title.lower() for title in train_titles}\n\ntrain_labels = set(train_metadata[\"dataset_label\"].unique())\ntrain_labels = {title.lower() for title in train_titles}\n\ntrain_datasets = train_titles.union(train_labels)\n\nextra_gov_datasets = set(pd.read_csv(\"../input/bigger-govt-dataset-list/data_set_800.csv\")[\"title\"].to_list())\nextra_gov_datasets = {dataset for dataset in extra_gov_datasets if not dataset.startswith(\"blog |\")}\n\nall_datasets = extra_gov_datasets.union(train_datasets)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:52.394736Z","iopub.execute_input":"2021-07-04T18:35:52.395166Z","iopub.status.idle":"2021-07-04T18:35:52.572667Z","shell.execute_reply.started":"2021-07-04T18:35:52.395133Z","shell.execute_reply":"2021-07-04T18:35:52.571665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Instantiate Pretrained Bert Model & Tokenizer","metadata":{}},{"cell_type":"code","source":"# BigBird roberta-base\nmodel_class, tokenizer_class, pretrained_weights = (BigBirdForTokenClassification, BigBirdTokenizerFast, '../input/huggingfacebigbirdrobertabase')\n\ntokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n\nlabel_list = [\"O\", \"B\", \"I\"]\nlabel2id = {label : id for id, label in enumerate(label_list)}\nid2label = {id : label for label, id in label2id.items()}\n\ndef get_pretrained_model(checkpoint=pretrained_weights):\n    config = BigBirdConfig(attention_type=\"block_sparse\", gradient_checkpointing=True, num_labels=3, id2label=id2label, label2id=label2id)\n    return model_class.from_pretrained(checkpoint, config=config)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:52.576896Z","iopub.execute_input":"2021-07-04T18:35:52.578929Z","iopub.status.idle":"2021-07-04T18:35:52.947531Z","shell.execute_reply.started":"2021-07-04T18:35:52.578889Z","shell.execute_reply":"2021-07-04T18:35:52.946641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Predictions","metadata":{}},{"cell_type":"code","source":"model = get_pretrained_model(\"../input/coleridgemodelcheckpoint\")\nmodel = model.to(device)\nmodel.eval()\n\nwith torch.no_grad():\n\n    rows = []\n    for filename in tqdm(os.listdir(testfiles_path)):\n        filepath = f\"{testfiles_path}{filename}\"\n        \n        # do string match first\n        found_labels = find_datasets_by_literal_matching(filepath, all_datasets)\n        \n        snippets = get_snippets_from_paper(filepath)\n        for snippet in snippets:\n            encoded_snippet = tokenizer(snippet, truncation=True, return_tensors=\"pt\")[\n                \"input_ids\"\n            ]\n            encoded_snippet = encoded_snippet.to(device)\n            model_outputs = model(encoded_snippet)\n            \n            snippet_preds = model_outputs[\"logits\"][0]\n            token_ids = encoded_snippet.squeeze()\n\n            # Remove ignored index (special tokens)\n            cleaned_snippet_preds = [\n                token_preds\n                for token_preds, token_id in zip(snippet_preds, token_ids)\n                if token_id not in tokenizer.all_special_ids\n            ]\n\n            predicted_tags = [\n                label_list[token_preds.argmax(0)]\n                for token_preds in cleaned_snippet_preds\n            ]\n\n            tokenized_snippet = tokenizer.convert_ids_to_tokens(token_ids, skip_special_tokens=True)\n\n            label_tokens = []\n            for token, tag in zip(tokenized_snippet, predicted_tags):\n                if tag == \"B\":\n                    label_tokens.append(token)\n                elif tag == \"I\" and len(label_tokens) > 0:\n                    label_tokens.append(token)\n                else:\n                    if len(label_tokens) > 0:\n                        found_label = tokenizer.convert_tokens_to_string(label_tokens)\n                        found_labels.add(clean_text(found_label))\n                        label_tokens = []\n\n        prediction_string = \"|\".join(sorted(found_labels))\n\n        rows.append({\"Id\": filename[:-5], \"PredictionString\": prediction_string})","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:35:52.948879Z","iopub.execute_input":"2021-07-04T18:35:52.949230Z","iopub.status.idle":"2021-07-04T18:36:38.478826Z","shell.execute_reply.started":"2021-07-04T18:35:52.949196Z","shell.execute_reply":"2021-07-04T18:36:38.478046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame(rows)\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T18:36:38.480848Z","iopub.execute_input":"2021-07-04T18:36:38.481212Z","iopub.status.idle":"2021-07-04T18:36:38.701063Z","shell.execute_reply.started":"2021-07-04T18:36:38.481175Z","shell.execute_reply":"2021-07-04T18:36:38.700265Z"},"trusted":true},"execution_count":null,"outputs":[]}]}