{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Installation\n- distilbert\n- distilroberta\n- distilgpt2 - can't do sequence classification\n- albert - better cv (.59 e1)\n- squeezebert\n- microsoft/xtremedistil-l6-h384-uncased\n- reformer\n- might as well try sota (xlnet, bert, albert, etc).\n- bert did well in cv, ok in leaderboard\n- scibert - 0.555 lb\n- roberta - 0.598cv, \n- deberta\n- xlnet - error running, it needs something else\n- try bert with adjusted weight decay - better generalization?","metadata":{}},{"cell_type":"code","source":"!pip install pytorch-crf","metadata":{"execution":{"iopub.status.busy":"2021-06-19T16:02:34.410363Z","iopub.execute_input":"2021-06-19T16:02:34.410993Z","iopub.status.idle":"2021-06-19T16:02:41.519327Z","shell.execute_reply.started":"2021-06-19T16:02:34.410859Z","shell.execute_reply":"2021-06-19T16:02:41.518336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport random\nimport re\nimport json\nimport gc\nfrom tqdm import tqdm\n\nfrom transformers import AutoModelForTokenClassification, AutoTokenizer, AutoConfig\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\n\nfrom torchcrf import CRF","metadata":{"execution":{"iopub.status.busy":"2021-06-19T16:02:41.520862Z","iopub.execute_input":"2021-06-19T16:02:41.52116Z","iopub.status.idle":"2021-06-19T16:02:43.360828Z","shell.execute_reply.started":"2021-06-19T16:02:41.521126Z","shell.execute_reply":"2021-06-19T16:02:43.359852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_TYPE = 'roberta-base'\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ntokenizer = AutoTokenizer.from_pretrained(MODEL_TYPE, add_prefix_space=True)\nconfig = AutoConfig.from_pretrained(MODEL_TYPE, add_prefix_space=True)\ntokenizer.save_pretrained(\"./tokenizer/\")\nconfig.save_pretrained('./tokenizer')","metadata":{"execution":{"iopub.status.busy":"2021-06-19T16:02:43.362156Z","iopub.execute_input":"2021-06-19T16:02:43.362396Z","iopub.status.idle":"2021-06-19T16:02:44.934653Z","shell.execute_reply.started":"2021-06-19T16:02:43.362373Z","shell.execute_reply":"2021-06-19T16:02:44.933691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing the data","metadata":{}},{"cell_type":"code","source":"# read in the csvs\ndf = pd.read_csv('../input/k/lichena/k/lichena/coleridge-pre-processing/data.csv')\nlabels_list = pd.read_csv('../input/k/lichena/k/lichena/coleridge-pre-processing/labels.csv').labels.unique().tolist()\nlabel_freq = {}\ny_true = {}\nwith open('../input/k/lichena/k/lichena/coleridge-pre-processing/frequencies.json', 'r') as f:\n    label_freq = json.load(f)\n\nwith open('../input/k/lichena/k/lichena/coleridge-pre-processing/y_true.json', 'r') as f:\n    y_true = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:20:27.857967Z","iopub.execute_input":"2021-06-17T16:20:27.858453Z","iopub.status.idle":"2021-06-17T16:20:42.093184Z","shell.execute_reply.started":"2021-06-17T16:20:27.858419Z","shell.execute_reply":"2021-06-17T16:20:42.092229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# {k: v for k, v in sorted(label_freq.items(), key=lambda item: item[1], reverse=True)}","metadata":{"execution":{"iopub.status.busy":"2021-06-17T21:04:00.061057Z","iopub.execute_input":"2021-06-17T21:04:00.061509Z","iopub.status.idle":"2021-06-17T21:04:00.066147Z","shell.execute_reply.started":"2021-06-17T21:04:00.061424Z","shell.execute_reply":"2021-06-17T21:04:00.065078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    text = re.sub('[^A-Za-z0-9()]+', ' ', str(txt)).strip()\n    return re.sub('\\s+', ' ', text)\n\ndef clean_label_result(label):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(label)).lower().strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:11.539349Z","iopub.execute_input":"2021-06-14T17:12:11.540043Z","iopub.status.idle":"2021-06-14T17:12:11.545401Z","shell.execute_reply.started":"2021-06-14T17:12:11.539982Z","shell.execute_reply":"2021-06-14T17:12:11.544346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenize the labels and create a map of label -> embedding\ndef tokenize_labels(labels):\n    label_set = {}\n    for label in labels:\n        label_set[label] = torch.tensor(tokenizer(label,max_length=512).input_ids[1:-1])\n    return label_set\n\ncleaned_labels = [clean_text(label) for label in labels_list]\nlabel_set = tokenize_labels(cleaned_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:11.546887Z","iopub.execute_input":"2021-06-14T17:12:11.5474Z","iopub.status.idle":"2021-06-14T17:12:11.681772Z","shell.execute_reply.started":"2021-06-14T17:12:11.547348Z","shell.execute_reply":"2021-06-14T17:12:11.68071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split labels into train/test groups\nnum_tests = 0\nx = 0\ntest_label_set = []\nwhile x < 150:\n    sample = random.sample(list(label_set), 1)[0]\n    if sample in label_freq:\n        if label_freq[sample] < 300:\n            num_tests += label_freq[sample]\n            test_label_set.append(sample)\n            x+=1","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:11.684105Z","iopub.execute_input":"2021-06-14T17:12:11.684572Z","iopub.status.idle":"2021-06-14T17:12:11.696174Z","shell.execute_reply.started":"2021-06-14T17:12:11.684526Z","shell.execute_reply":"2021-06-14T17:12:11.69461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(test_label_set).to_csv('test_label_set.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:11.699826Z","iopub.execute_input":"2021-06-14T17:12:11.700235Z","iopub.status.idle":"2021-06-14T17:12:11.710836Z","shell.execute_reply.started":"2021-06-14T17:12:11.700189Z","shell.execute_reply":"2021-06-14T17:12:11.709567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_dataset(df):\n    train_tp_texts = []\n    train_tp_labels = []\n    train_fp_texts = []\n    train_tp_ids = []\n    train_fp_ids = []\n    test_tp_texts = []\n    test_tp_labels = []\n    test_fp_texts = []\n    test_tp_ids = []\n    test_fp_ids = []\n    for index, row in tqdm(df.iterrows(), total=len(df)):\n        chunk, labels = row['chunks'], row['labels']\n        if labels == labels:\n            is_train = True\n            for label in labels.split(\"|\"):\n                if label in test_label_set:\n                    is_train=False\n            if is_train:\n                train_tp_texts.append(chunk)\n                train_tp_labels.append(labels.split(\"|\"))\n                train_tp_ids.append(row['ids'])\n            else:\n                test_tp_texts.append(chunk)\n                test_tp_labels.append(labels.split(\"|\"))\n                test_tp_ids.append(row['ids'])   \n        else:\n            if random.random() > 0.5:\n                train_fp_texts.append(chunk)\n                train_fp_ids.append(row['ids'])\n            else:\n                test_fp_texts.append(chunk)\n                test_fp_ids.append(row['ids'])\n    return train_tp_texts, train_tp_labels, train_fp_texts, train_tp_ids, train_fp_ids, test_tp_texts, test_tp_labels, test_fp_texts, test_tp_ids, test_fp_ids\ntrain_tp_texts, train_tp_labels, train_fp_texts, train_tp_ids, train_fp_ids, test_tp_texts, test_tp_labels, test_fp_texts, test_tp_ids, test_fp_ids = generate_dataset(df)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:11.714788Z","iopub.execute_input":"2021-06-14T17:12:11.71516Z","iopub.status.idle":"2021-06-14T17:12:16.7669Z","shell.execute_reply.started":"2021-06-14T17:12:11.715124Z","shell.execute_reply":"2021-06-14T17:12:16.765863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_text(labels, parent_array):\n    idxs = torch.zeros(len(parent_array))\n    for sub_array in labels:\n        sub_len = len(sub_array)\n        for idx, e in enumerate(parent_array):\n            if e == sub_array[0]:\n                if sub_len == len(parent_array[idx:idx+sub_len]):\n                    if torch.all(parent_array[idx:idx+sub_len].eq(sub_array)):\n                        idxs[idx] = 1\n                        if sub_len > 1:\n                            idxs[range(idx+1, idx+sub_len)] = 2\n    return idxs","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:16.768595Z","iopub.execute_input":"2021-06-14T17:12:16.769031Z","iopub.status.idle":"2021-06-14T17:12:16.777173Z","shell.execute_reply.started":"2021-06-14T17:12:16.768985Z","shell.execute_reply":"2021-06-14T17:12:16.776453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ColeridgeDataset(Dataset):\n    def __init__(self, tp_texts, tp_labels, fp_texts, tp_ids, fp_ids, mode='train', tp_frac=0.5):\n        self.tp_texts = tp_texts\n        self.tp_labels = tp_labels\n        self.fp_texts = fp_texts\n        self.tp_frac = tp_frac\n        self.tp_ids = tp_ids\n        self.fp_ids = fp_ids\n        self.mode = mode\n        \n    def __len__(self):\n        return int(1/self.tp_frac * len(self.tp_texts))\n\n    def __getitem__(self, idx):\n        labels = []\n        if idx < len(self.tp_texts):\n            text = self.tp_texts[idx]\n            labels = self.tp_labels[idx]\n            _id = self.tp_ids[idx]\n            weight = [1 / label_freq[l] for l in labels]\n            weight = sum(weight) / len(weight)\n        else:\n            index = random.randint(0, len(self.fp_texts)-1)\n            text = self.fp_texts[index]\n            _id = self.fp_ids[idx]\n            weight = 0.1\n        return text, labels, _id, weight\n\ndef collate_fn(batch):\n    texts = [item[0] for item in batch]\n    ids = [item[2] for item in batch]\n    weights = [item[3] for item in batch]\n    encoding = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=512)\n    input_ids = encoding.input_ids\n    labels = torch.zeros(input_ids.shape[0], input_ids.shape[1]) # outer, start, end, inner\n    text_indices = []\n    for idx, item in enumerate(batch):\n        dataset_titles = item[1]\n        if dataset_titles:\n            dataset_toks = [label_set[label] for label in dataset_titles if label != 'labels']\n            labels[idx] = label_text(dataset_toks, input_ids[idx])\n    return input_ids, encoding.attention_mask, labels.type(torch.LongTensor),  ids, torch.tensor(weights)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:16.778138Z","iopub.execute_input":"2021-06-14T17:12:16.778485Z","iopub.status.idle":"2021-06-14T17:12:16.795525Z","shell.execute_reply.started":"2021-06-14T17:12:16.778449Z","shell.execute_reply":"2021-06-14T17:12:16.794481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = ColeridgeDataset(train_tp_texts, train_tp_labels, train_fp_texts, train_tp_ids, train_fp_ids, mode='train', tp_frac=0.8)\ntrain_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True, collate_fn=collate_fn)\n\ntest_dataset = ColeridgeDataset(test_tp_texts, test_tp_labels, test_fp_texts, test_tp_ids, test_fp_ids, mode='test', tp_frac=0.2)\ntest_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:16.796675Z","iopub.execute_input":"2021-06-14T17:12:16.796971Z","iopub.status.idle":"2021-06-14T17:12:16.812203Z","shell.execute_reply.started":"2021-06-14T17:12:16.796944Z","shell.execute_reply":"2021-06-14T17:12:16.811396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = 0\nfor batch in train_dataloader:\n    input_ids, attention_mask, labels, ids, weight = batch\n    print(labels[0])\n    break","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:16.815843Z","iopub.execute_input":"2021-06-14T17:12:16.816303Z","iopub.status.idle":"2021-06-14T17:12:17.025205Z","shell.execute_reply.started":"2021-06-14T17:12:16.816268Z","shell.execute_reply":"2021-06-14T17:12:17.024161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing and defining the model","metadata":{}},{"cell_type":"code","source":"model = AutoModelForTokenClassification.from_pretrained(MODEL_TYPE, num_labels=3)\ncrf = CRF(3, batch_first=True)\n# checkpoint = torch.load('../input/coleridge-ner/checkpoint.pt', map_location=DEVICE)\n# model = checkpoint['model']\n# crf = checkpoint['crf']","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:17.026457Z","iopub.execute_input":"2021-06-14T17:12:17.026733Z","iopub.status.idle":"2021-06-14T17:12:17.42129Z","shell.execute_reply.started":"2021-06-14T17:12:17.026707Z","shell.execute_reply":"2021-06-14T17:12:17.42036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T17:12:17.42451Z","iopub.execute_input":"2021-06-14T17:12:17.424975Z","iopub.status.idle":"2021-06-14T17:12:17.429957Z","shell.execute_reply.started":"2021-06-14T17:12:17.424929Z","shell.execute_reply":"2021-06-14T17:12:17.428764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stops = stopwords.words('english')\nstops.append('[sep]')\nstops.append('[PAD]')\nstops.append('PAD')\nstops.append('pad')\n\ndef clean_front(split):\n    for idx, e in enumerate(split):\n        if e not in stops:\n            return split[idx:]\n        \ndef clean_back(split):\n    for i in reversed(range(len(split))):\n        if split[i] not in stops:\n            return split[:i+1]\n        \ndef clean_result(title):\n#     title = title.lower()\n    title = title.replace('<s>', '')\n    title = title.replace('</s>', '')\n    split = title.split()\n    if split:\n        if len(split[-1]) <= 2:\n            split = split[:-1]\n    if split:\n        split = clean_front(split)\n    if split:\n        split = clean_back(split)\n    if split:\n        title = ' '.join(split)\n        return title","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T17:29:31.891482Z","iopub.execute_input":"2021-06-14T17:29:31.891838Z","iopub.status.idle":"2021-06-14T17:29:31.903785Z","shell.execute_reply.started":"2021-06-14T17:29:31.891809Z","shell.execute_reply":"2021-06-14T17:29:31.90258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_fbeta(y_true,\n                  y_pred,\n                  beta = 0.5) -> float:\n    \"\"\"Compute the Jaccard-based micro FBeta score.\n\n    References\n    ----------\n    - https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\n    \"\"\"\n\n    fp_list = []\n    tp_list = []\n    fn_list = []\n    def _jaccard_similarity(str1: str, str2: str) -> float:\n        a = set(str1.split()) \n        b = set(str2.split())\n        c = a.intersection(b)\n        return float(len(c)) / (len(a) + len(b) - len(c))\n\n    tp = 0  # true positive\n    fp = 0  # false positive\n    fn = 0  # false negative\n    for ground_truth_list, predicted_string_list in zip(y_true, y_pred):\n        predicted_string_list_sorted = sorted(predicted_string_list)\n        if len(ground_truth_list) == 0 and len(predicted_string_list_sorted) > 0:\n            fp += len(predicted_string_list_sorted)\n            fp_list += predicted_string_list_sorted\n        else:\n            for ground_truth in sorted(ground_truth_list):\n                if len(predicted_string_list_sorted) == 0:\n                    fn += 1\n                    fn_list.append(ground_truth)\n                else:\n                    similarity_scores = [\n                        _jaccard_similarity(ground_truth, predicted_string)\n                        for predicted_string in predicted_string_list_sorted\n                    ]\n                    matched_idx = np.argmax(similarity_scores)\n                    if similarity_scores[matched_idx] >= 0.5:\n                        tp_list.append(predicted_string_list_sorted[matched_idx])\n                        predicted_string_list_sorted.pop(matched_idx)\n                        tp += 1\n                    else:\n                        fn_list.append(ground_truth)\n                        fn += 1\n            fp += len(predicted_string_list_sorted)\n            fp_list += predicted_string_list_sorted\n\n    tp *= (1 + beta ** 2)\n    fn *= beta ** 2\n    fbeta_score = tp / (tp + fp + fn)\n    print('fp: ',fp, '\\ttp: ',tp, '\\tfn: ', fn)\n    return fbeta_score, fp_list, fn_list, tp_list","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-14T17:32:47.004696Z","iopub.execute_input":"2021-06-14T17:32:47.005229Z","iopub.status.idle":"2021-06-14T17:32:47.015566Z","shell.execute_reply.started":"2021-06-14T17:32:47.005178Z","shell.execute_reply":"2021-06-14T17:32:47.01459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lcs(X, Y):\n    m = len(X)\n    n = len(Y)\n \n    # Create a table to store lengths of\n    # longest common suffixes of substrings.\n    # Note that LCSuff[i][j] contains length\n    # of longest common suffix of X[0..i-1] and\n    # Y[0..j-1]. The first row and first\n    # column entries have no logical meaning,\n    # they are used only for simplicity of program\n    LCSuff = [[0 for i in range(n + 1)]\n                 for j in range(m + 1)]\n \n    # To store length of the\n    # longest common substring\n    length = 0\n \n    # To store the index of the cell\n    # which contains the maximum value.\n    # This cell's index helps in building\n    # up the longest common substring\n    # from right to left.\n    row, col = 0, 0\n \n    # Following steps build LCSuff[m+1][n+1]\n    # in bottom up fashion.\n    for i in range(m + 1):\n        for j in range(n + 1):\n            if i == 0 or j == 0:\n                LCSuff[i][j] = 0\n            elif X[i - 1] == Y[j - 1]:\n                LCSuff[i][j] = LCSuff[i - 1][j - 1] + 1\n                if length < LCSuff[i][j]:\n                    length = LCSuff[i][j]\n                    row = i\n                    col = j\n            else:\n                LCSuff[i][j] = 0\n \n    # if true, then no common substring exists\n    if length == 0:\n        return\n \n    # allocate space for the longest\n    # common substring\n    resultStr = ['0'] * length\n \n    # traverse up diagonally form the\n    # (row, col) cell until LCSuff[row][col] != 0\n    while LCSuff[row][col] != 0:\n        length -= 1\n        resultStr[length] = X[row - 1] # or Y[col-1]\n \n        # move diagonally up to previous cell\n        row -= 1\n        col -= 1\n \n    # required longest common substring\n    return ''.join(resultStr)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:29:33.685059Z","iopub.execute_input":"2021-06-14T17:29:33.685493Z","iopub.status.idle":"2021-06-14T17:29:33.69705Z","shell.execute_reply.started":"2021-06-14T17:29:33.685453Z","shell.execute_reply":"2021-06-14T17:29:33.695556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score_results(titles, scores, ids, threshold = 0.95):\n    results = {}\n    actual = {}\n    for _id in pd.Series(test_tp_ids + test_fp_ids).unique():\n        results[_id] = []\n    for idx, _id in enumerate(test_tp_ids):\n        if _id not in actual:\n            actual[_id] = []\n        actual[_id] += [clean_label_result(l) for l in test_tp_labels[idx]]\n    for idx, _id in enumerate(tqdm(ids)):\n        title = titles[idx]\n        if len(title) > 2 and scores[idx] > threshold:\n            cleaned = clean_result(clean_label_result(title))\n            if cleaned and cleaned not in results[_id] and ' ' in cleaned:\n                is_new = True\n                for idx, element in enumerate(results[_id]):\n                    if element in cleaned:\n                        is_new = False\n                        results[_id][idx] = cleaned\n                    elif cleaned in element:\n                        is_new = False\n                        results[_id][idx] = element\n                if is_new:\n                    results[_id].append(cleaned)\n    y_pred = []\n    target = []\n    for _id in pd.Series(test_tp_ids + test_fp_ids).unique():\n        y_pred.append(results[_id])\n        if _id in actual:\n            target.append(actual[_id])\n        else:\n            target.append([])\n    fbeta, fp_list, fn_list, tp_list = compute_fbeta(target, y_pred)\n    return fbeta, fp_list, fn_list, tp_list","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:32:57.34875Z","iopub.execute_input":"2021-06-14T17:32:57.349116Z","iopub.status.idle":"2021-06-14T17:32:57.36Z","shell.execute_reply.started":"2021-06-14T17:32:57.349084Z","shell.execute_reply":"2021-06-14T17:32:57.359012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_titles(input_ids, pred, score, _id):\n    titles = []\n    scores = []\n    ids = []\n    for idx in range(input_ids.shape[0]):\n        is_title = False\n        tmp_toks = []\n        tmp_scores = []\n        for row in range(input_ids.shape[1]):\n            if pred[idx][row] > 0:\n                if is_title and pred[idx][row] == 1:\n                    titles.append(tokenizer.decode(tmp_toks).strip())\n                    scores.append((sum(tmp_scores) / len(tmp_scores)).item())\n                    ids.append(_id[idx])\n                    tmp_toks = []\n                    tmp_scores = []\n                tmp_toks.append(input_ids[idx][row])\n                tmp_scores.append(score[idx][row])\n                is_title = True\n            elif is_title and (pred[idx][row] == 0 or row == input_ids.shape[1] - 1 or input_ids[idx][row] == 102):\n                is_title = False\n                titles.append(tokenizer.decode(tmp_toks).strip())\n                scores.append((sum(tmp_scores) / len(tmp_scores)).item())\n                ids.append(_id[idx])\n                tmp_toks = []\n                tmp_scores = []\n            elif is_title:\n                tmp_toks.append(input_ids[idx][row])\n                tmp_scores.append(score[idx][row])\n    return titles, scores, ids","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:32:57.746042Z","iopub.execute_input":"2021-06-14T17:32:57.746644Z","iopub.status.idle":"2021-06-14T17:32:57.757658Z","shell.execute_reply.started":"2021-06-14T17:32:57.746608Z","shell.execute_reply":"2021-06-14T17:32:57.756891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(dataloader, model, optimizer, scheduler):\n    gc.collect()\n    model.train()\n    loader = tqdm(dataloader)\n    avg_loss = 0\n    for idx, batch in enumerate(loader):\n        gc.collect()\n        input_ids, attention_mask, labels, ids, weight = batch\n        labels = labels.to(DEVICE)\n        input_ids = input_ids.to(DEVICE)\n        attention_mask = attention_mask.type(torch.uint8).to(DEVICE)\n        weight = weight.to(DEVICE)\n        emissions = model(input_ids, attention_mask=attention_mask).logits\n        loss = -crf(emissions, labels, mask=attention_mask, reduction='none')\n        loss = (loss * weight).sum()\n        loss.backward()\n        optimizer.step()\n        if scheduler:\n            scheduler.step()\n        optimizer.zero_grad()\n        if idx % 50 == 0:\n            print(\"Step = \",idx,\"loss = \",loss.detach().item())\n        avg_loss += loss.detach().item()\n    return avg_loss / len(dataloader)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:32:58.415743Z","iopub.execute_input":"2021-06-14T17:32:58.416243Z","iopub.status.idle":"2021-06-14T17:32:58.425042Z","shell.execute_reply.started":"2021-06-14T17:32:58.416211Z","shell.execute_reply":"2021-06-14T17:32:58.42373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_fn(dataloader, model):\n    gc.collect()\n    model.eval()\n    with torch.no_grad():\n        titles = []\n        scores = []\n        ids = []\n        loader = tqdm(dataloader)\n        for batch in loader:\n            gc.collect()\n            input_ids, attention_mask, labels, _id, weight = batch\n            input_ids = input_ids.to(DEVICE)\n            attention_mask = attention_mask.type(torch.uint8).to(DEVICE)\n            emissions = model(input_ids, attention_mask=attention_mask).logits\n            pred = torch.tensor(crf.decode(emissions))\n            score = torch.max(torch.softmax(emissions.permute(0,2,1), dim=1), dim=1).values\n            _titles, _scores, _ids = get_titles(input_ids, pred, score, _id)\n            titles += _titles\n            scores += _scores\n            ids += _ids\n        return titles, scores, ids","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:29:37.194226Z","iopub.execute_input":"2021-06-14T17:29:37.194626Z","iopub.status.idle":"2021-06-14T17:29:37.204101Z","shell.execute_reply.started":"2021-06-14T17:29:37.19459Z","shell.execute_reply":"2021-06-14T17:29:37.2028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import get_scheduler\n\nNUM_EPOCHS = 3\nmodel.to(DEVICE)\ncrf.to(DEVICE)\nparams = list(model.parameters()) + list(crf.parameters())\noptimizer = torch.optim.AdamW(params, lr=3e-5, weight_decay=0.1)\n\nnum_training_steps = NUM_EPOCHS * len(train_dataloader)\nlr_scheduler = get_scheduler(\n    \"linear\",\n    optimizer=optimizer,\n    num_warmup_steps=0.1 * len(train_dataloader),\n    num_training_steps=num_training_steps\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:17.545074Z","iopub.execute_input":"2021-06-14T17:12:17.545356Z","iopub.status.idle":"2021-06-14T17:12:17.561484Z","shell.execute_reply.started":"2021-06-14T17:12:17.545331Z","shell.execute_reply":"2021-06-14T17:12:17.560358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for e in range(NUM_EPOCHS):\n    try:\n        flag = 0\n        best_fbeta = -1\n        fp_list = []\n        fn_list = []\n        tp_list = []\n#         train_loss = train_fn(train_dataloader, model, optimizer, None)\n        train_loss = train_fn(train_dataloader, model, optimizer, lr_scheduler)\n        gc.collect()\n        titles, scores, ids = eval_fn(test_dataloader, model)\n        gc.collect()\n        thresholds = [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n        thresh_fbeta = 0\n        thresh = 0\n        for t in thresholds:\n            fbeta, fp_list, fn_list, tp_list = score_results(titles, scores, ids, t)\n            if fbeta >= thresh_fbeta:\n                thresh_fbeta = fbeta\n                thresh = t\n        print(\"fbeta:\\t\" + str(thresh_fbeta))\n        if thresh_fbeta >= best_fbeta:\n            best_fbeta = thresh_fbeta\n            checkpoint = {\n                'fbeta': best_fbeta,\n                'thresh': thresh,\n                'model': model,\n                'crf': crf,\n                'optimizer_state_dict': optimizer.state_dict(),\n                'epoch': e\n            }\n            torch.save(checkpoint, \"./checkpoint.pt\")\n            pd.Series(fp_list).to_csv('fp_list.csv', index=False)\n            pd.Series(fn_list).to_csv('fn_list.csv', index=False)\n            pd.Series(tp_list).to_csv('tp_list.csv', index=False)\n        else:\n            flag +=1\n            if (flag > 2):\n                break\n    except Exception as exception:\n        print(exception)\n        checkpoint = {\n            'fbeta': 0,\n            'thresh': 0,\n            'model': model,\n            'crf': crf,\n            'optimizer_state_dict': optimizer.state_dict(),\n            'epoch': e\n        }\n        torch.save(checkpoint, \"./checkpoint.pt\")","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:12:17.563843Z","iopub.execute_input":"2021-06-14T17:12:17.564284Z","iopub.status.idle":"2021-06-14T17:29:25.63187Z","shell.execute_reply.started":"2021-06-14T17:12:17.564178Z","shell.execute_reply":"2021-06-14T17:29:25.628135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}