{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip install -Uqq datasets seqeval s3fs","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:21.322588Z","iopub.execute_input":"2021-06-10T10:17:21.322918Z","iopub.status.idle":"2021-06-10T10:17:21.326968Z","shell.execute_reply.started":"2021-06-10T10:17:21.322843Z","shell.execute_reply":"2021-06-10T10:17:21.326201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:21.332597Z","iopub.execute_input":"2021-06-10T10:17:21.332884Z","iopub.status.idle":"2021-06-10T10:17:22.005066Z","shell.execute_reply.started":"2021-06-10T10:17:21.332857Z","shell.execute_reply":"2021-06-10T10:17:22.004097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport json\nfrom glob import glob\nimport itertools\n\nimport pandas as pd\nimport numpy as np\nfrom transformers import TrainingArguments, Trainer, pipeline, EarlyStoppingCallback\n#from transformers import AutoTokenizer as Tokenizer, AutoModelForTokenClassification as Model\n#from transformers import RobertaTokenizerFast as Tokenizer, RobertaForTokenClassification as Model\nfrom transformers import DistilBertTokenizerFast as Tokenizer, DistilBertForTokenClassification as Model\nimport torch\nfrom tqdm import tqdm, notebook\nfrom sklearn.metrics import fbeta_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T10:17:22.006588Z","iopub.execute_input":"2021-06-10T10:17:22.006871Z","iopub.status.idle":"2021-06-10T10:17:29.399201Z","shell.execute_reply.started":"2021-06-10T10:17:22.006842Z","shell.execute_reply":"2021-06-10T10:17:29.398398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples = 10000\neval_samples = 200","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:29.400914Z","iopub.execute_input":"2021-06-10T10:17:29.401240Z","iopub.status.idle":"2021-06-10T10:17:29.408533Z","shell.execute_reply.started":"2021-06-10T10:17:29.401211Z","shell.execute_reply":"2021-06-10T10:17:29.407612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower()).strip()\n\ndef prep_labels(path):\n    short_labels = ['agid', 'anss', 'blsa', 'charybdis', 'cord 19', 'nces', 'c cap', 'bbs', 'ricord', 'niagads']\n    cleaned_labels = pd.read_csv(path)['cleaned_label'].str.strip().unique().tolist()\n    cleaned_labels = sorted(cleaned_labels + short_labels)\n    s_labels = pd.Series(cleaned_labels, name='label')\n    return s_labels\n\n\ndef prep_data(fns, labels):\n    # JSON to WORDS & LABELS\n    data = {'tokens': [], 'labels': []}\n    for fn in tqdm(fns):\n        with open(fn, 'r') as f:\n            obj = json.load(f)\n        for section in obj:\n            text = clean_text(section.get('text'))\n            if text:\n                text_labels = list(re.sub(r'[^ ]', '0', text))\n                for label in labels:\n                    matches = [m.span() for m in re.finditer(label, text)]\n                    for m in matches:\n                        text_labels[m[0]:m[1]] = re.sub(r'[^ ]', '1', label)\n                text_labels = ['1' in e for e in ''.join(text_labels).split()]\n                text_words = text.split(' ')\n                assert len(text_words) == len(text_labels)\n                while len(text_words) > 0:\n                    labels_np = np.array(text_labels[:512]).astype(int)\n                    # ONLY ADD TEXT WITH AT LEAST ONE LABEL\n                    if labels_np.sum() > 0:\n                        data['tokens'].append(text_words[:512])\n                        data['labels'].append(labels_np)\n                    text_words = text_words[500:]\n                    text_labels = text_labels[500:]\n    return data\n\ndef tokenize_and_align_labels(data, tokenizer, label_all_tokens=True):\n    # Tokenize Words and align Labels\n    tokenized_inputs = tokenizer(data['tokens'], truncation=True, is_split_into_words=True, padding=True)\n\n    labels = []\n    for i, label in enumerate(data['labels']):\n        word_ids = tokenized_inputs.word_ids(batch_index=i)\n        previous_word_idx = None\n        label_ids = []\n        for word_idx in word_ids:\n            # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n            # ignored in the loss function.\n            if word_idx is None:\n                label_ids.append(-100)\n            # We set the label for the first token of each word.\n            elif word_idx != previous_word_idx:\n                label_ids.append(label[word_idx])\n            # For the other tokens in a word, we set the label to either the current label or -100, depending on\n            # the label_all_tokens flag.\n            else:\n                label_ids.append(label[word_idx] if label_all_tokens else -100)\n            previous_word_idx = word_idx\n\n        labels.append(label_ids)\n\n    tokenized_inputs['labels'] = labels\n    return tokenized_inputs\n\nclass CRDataset(torch.utils.data.Dataset):\n    def __init__(self, json_path, labels_path, tokenizer):\n        cleaned_labels = prep_labels(labels_path)\n        data = prep_data(json_path, cleaned_labels)\n        tokenized_data = tokenize_and_align_labels(data, tokenizer)\n        self.encodings = tokenized_data['input_ids']\n        self.labels = tokenized_data['labels']\n\n    def __getitem__(self, idx):\n        item = {\n            'input_ids': torch.tensor(self.encodings[idx]),\n            'labels': torch.tensor(self.labels[idx])\n        }\n        return item\n\n    def __len__(self):\n        return len(self.labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:29.410299Z","iopub.execute_input":"2021-06-10T10:17:29.410662Z","iopub.status.idle":"2021-06-10T10:17:29.428859Z","shell.execute_reply.started":"2021-06-10T10:17:29.410626Z","shell.execute_reply":"2021-06-10T10:17:29.427657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_path = '/kaggle/input/hfdistilbertbaseuncasedtokenclassification'\ncr_tokenizer = Tokenizer.from_pretrained(pretrained_path, padding=\"max_length\", truncation=True, max_length=512, return_tensors=\"pt\", add_prefix_space=True)\ncr_model = Model.from_pretrained(pretrained_path, num_labels=2)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:29.430383Z","iopub.execute_input":"2021-06-10T10:17:29.430811Z","iopub.status.idle":"2021-06-10T10:17:37.157506Z","shell.execute_reply.started":"2021-06-10T10:17:29.430772Z","shell.execute_reply":"2021-06-10T10:17:37.156621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns = glob('/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json')\ntrain_fns = fns[:train_samples]\ntrain_dataset = CRDataset(train_fns, '/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv', cr_tokenizer)\nif eval_samples:\n    eval_fns = fns[train_samples:train_samples+eval_samples]\n    val_dataset = CRDataset(eval_fns, '/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv', cr_tokenizer)\nelse:\n    val_dataset = None","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:17:37.158824Z","iopub.execute_input":"2021-06-10T10:17:37.159167Z","iopub.status.idle":"2021-06-10T10:18:39.350021Z","shell.execute_reply.started":"2021-06-10T10:17:37.159129Z","shell.execute_reply":"2021-06-10T10:18:39.348990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\ntraining_args = TrainingArguments(\n    #overwrite_output_dir=True,\n    output_dir='/kaggle/tmp/results', # output directory\n    num_train_epochs=3,               # total number of training epochs\n    per_device_train_batch_size=16,   # batch size per device during training\n    per_device_eval_batch_size=64,    # batch size for evaluation\n    warmup_steps=500,                 # number of warmup steps for learning rate scheduler\n    weight_decay=0.01,                # strength of weight decay\n    logging_dir='/kaggle/tmp/logs',   # directory for storing logs\n    logging_steps=50,\n    report_to='none',\n    evaluation_strategy='steps',\n    load_best_model_at_end=True\n)\n\ntrainer = Trainer(\n    model=cr_model,                      # the instantiated 🤗 Transformers model to be trained\n    args=training_args,                  # training arguments, defined above\n    train_dataset=train_dataset,         # training dataset\n    eval_dataset=val_dataset,            # evaluation dataset\n    callbacks=[EarlyStoppingCallback(10)]\n)\n\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:18:39.352378Z","iopub.execute_input":"2021-06-10T10:18:39.352734Z","iopub.status.idle":"2021-06-10T10:41:49.221432Z","shell.execute_reply.started":"2021-06-10T10:18:39.352696Z","shell.execute_reply":"2021-06-10T10:41:49.220457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.save_pretrained('./model')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:41:49.224895Z","iopub.execute_input":"2021-06-10T10:41:49.225219Z","iopub.status.idle":"2021-06-10T10:41:49.233444Z","shell.execute_reply.started":"2021-06-10T10:41:49.225183Z","shell.execute_reply":"2021-06-10T10:41:49.232557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = pipeline('ner', model=cr_model, tokenizer=cr_tokenizer, device=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:41:49.235748Z","iopub.execute_input":"2021-06-10T10:41:49.236054Z","iopub.status.idle":"2021-06-10T10:41:49.691062Z","shell.execute_reply.started":"2021-06-10T10:41:49.236021Z","shell.execute_reply":"2021-06-10T10:41:49.688946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunked_iterable(iterable, size):\n    it = iter(iterable)\n    while True:\n        chunk = tuple(itertools.islice(it, size))\n        if not chunk:\n            break\n        yield chunk\n\ndef classify_sample(fn):\n    text = []\n    with open(fn, 'r') as f:\n        for section in json.load(f):\n            if section and section.get('text'):\n                text.append(section['text'])\n    return '|'.join(label_text(' '.join(text)))\n    \ndef label_text(text):\n    labels = []\n    for chunk in chunked_iterable(text.split(), 512):\n        results = classifier(' '.join(chunk))\n        tmp = []\n        for n, r in enumerate(results):\n            if r['entity'] == 'LABEL_1' and r['word'] != '.' and r['score'] > 0.8:\n                print(r['word'], r['start'], r['end'], r['score'])\n                try:\n                    if n < len(results) and results[n+1]['entity'] == 'LABEL_1':\n                        if r['end'] + 1 == results[n+1]['start'] or r['end'] == results[n+1]['start']:\n                            tmp.append(r['word'].replace(' .', ''))\n                    if n < len(results) and results[n+1]['entity'] != 'LABEL_1':\n                        if tmp:\n                            tmp.append(r['word'].replace(' .', ''))\n                            l = ' '.join(tmp)\n                            labels.append(l.replace(' ##', ''))\n                            tmp = []\n                        else:\n                            labels.append(r['word'].replace(' .', ''))\n                except:\n                    labels.append(r['word'])\n    labels = [l for l in labels if not l.endswith('nces')]\n    return list(set(labels))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:15.215931Z","iopub.execute_input":"2021-06-10T10:52:15.216259Z","iopub.status.idle":"2021-06-10T10:52:15.228595Z","shell.execute_reply.started":"2021-06-10T10:52:15.216226Z","shell.execute_reply":"2021-06-10T10:52:15.227488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns[50], fns[10004]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:15.556358Z","iopub.execute_input":"2021-06-10T10:52:15.556674Z","iopub.status.idle":"2021-06-10T10:52:15.562305Z","shell.execute_reply.started":"2021-06-10T10:52:15.556644Z","shell.execute_reply":"2021-06-10T10:52:15.561370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\nt[t['Id'] == '11cd3b53-957d-4eb0-bcdf-652d34e63af4']['cleaned_label'].unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:15.846985Z","iopub.execute_input":"2021-06-10T10:52:15.847318Z","iopub.status.idle":"2021-06-10T10:52:15.902918Z","shell.execute_reply.started":"2021-06-10T10:52:15.847266Z","shell.execute_reply":"2021-06-10T10:52:15.901936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classify_sample(fns[50])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:16.221350Z","iopub.execute_input":"2021-06-10T10:52:16.221645Z","iopub.status.idle":"2021-06-10T10:52:16.403963Z","shell.execute_reply.started":"2021-06-10T10:52:16.221617Z","shell.execute_reply":"2021-06-10T10:52:16.403222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t[t['Id'] == '1d089481-7727-4c6d-a641-72870cb0a0fa']['cleaned_label'].unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:16.637880Z","iopub.execute_input":"2021-06-10T10:52:16.638211Z","iopub.status.idle":"2021-06-10T10:52:16.649966Z","shell.execute_reply.started":"2021-06-10T10:52:16.638177Z","shell.execute_reply":"2021-06-10T10:52:16.649030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classify_sample(fns[10004])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:17.309694Z","iopub.execute_input":"2021-06-10T10:52:17.310014Z","iopub.status.idle":"2021-06-10T10:52:17.583422Z","shell.execute_reply.started":"2021-06-10T10:52:17.309983Z","shell.execute_reply":"2021-06-10T10:52:17.582666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submissions are evaluated on a Jaccard-based FBeta score between predicted texts and ground truth texts, with Beta = 0.5 (a micro F0.5 score). Multiple predictions are delineated with a pipe (|) character in the submission file.\n\nThe following is Python code for calculating the Jaccard score for a single prediction string against a single ground truth string. Note that the overall score for a sample uses Jaccard to compare multiple ground truth and prediction strings that are pipe-delimited - this code does not handle that process or the final micro F-beta calculation.","metadata":{}},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split())\n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\ndef score(y_true, y_pred):\n    y_true_l = sorted(y_true.split('|'))\n    y_pred_l = sorted(y_pred.split('|'))\n    tp, fp, fn = 0, 0, 0\n    for pred_label in y_pred_l:\n        max_s = 0\n        matched_ix = None\n        for n, true_label in enumerate(y_true_l):\n            s = jaccard(true_label, pred_label)\n            if s >= 0.5:\n                max_s = s\n                matched_ix = n\n        if max_s:\n            tp += 1\n        else:\n            pass\n    return fbeta_score(y_true, y_pred, average='micro', beta=0.5)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:52:40.366829Z","iopub.execute_input":"2021-06-10T10:52:40.367180Z","iopub.status.idle":"2021-06-10T10:52:40.376185Z","shell.execute_reply.started":"2021-06-10T10:52:40.367146Z","shell.execute_reply":"2021-06-10T10:52:40.375391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each publication's set of predictions, a token-based Jaccard score is calculated for each potential prediction / ground truth pair. The prediction with the highest score for a given ground truth is matched with that ground truth.\n\nPredicted strings for each publication are sorted alphabetically and processed in that order. Any scoring ties are resolved on the basis of that sort.\nAny matched predictions where the Jaccard score meets or exceeds the threshold of 0.5 are counted as true positives (TP), the remainder as false positives (FP).\nAny unmatched predictions are counted as false positives (FP).\nAny ground truths with no nearest predictions are counted as false negatives (FN).\nAll TP, FP and FN across all samples are used to calculate a final micro F0.5 score. (Note that a micro F score does precisely this, creating one pool of TP, FP and FN that is used to calculate a score for the entire set of predictions.)","metadata":{}},{"cell_type":"code","source":"test_fns = glob('/kaggle/input/coleridgeinitiative-show-us-the-data/test/*.json')\n#test_fns = glob('/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json')\ntest_sub = []\nfor fn in tqdm(test_fns):\n    test_id = fn.rsplit('/', 1)[1].split('.')[0]\n    labels = classify_sample(fn)\n    test_sub.append({'Id': test_id, 'PredictionString': labels})\ntest_df = pd.DataFrame(test_sub)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:41:50.532442Z","iopub.status.idle":"2021-06-10T10:41:50.533010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:41:50.534313Z","iopub.status.idle":"2021-06-10T10:41:50.535134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls *.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:41:50.536815Z","iopub.status.idle":"2021-06-10T10:41:50.537406Z"},"trusted":true},"execution_count":null,"outputs":[]}]}