{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/pytorchcrf/pytorch_crf-0.7.2-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-20T20:40:42.785039Z","iopub.execute_input":"2021-06-20T20:40:42.785496Z","iopub.status.idle":"2021-06-20T20:41:10.931081Z","shell.execute_reply.started":"2021-06-20T20:40:42.785437Z","shell.execute_reply":"2021-06-20T20:41:10.929646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport random\nimport re\nimport json\nfrom tqdm import tqdm\n\nfrom transformers import AutoModelForTokenClassification, AutoTokenizer\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torchcrf import CRF\n\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2021-06-20T20:41:10.933517Z","iopub.execute_input":"2021-06-20T20:41:10.933955Z","iopub.status.idle":"2021-06-20T20:41:14.542008Z","shell.execute_reply.started":"2021-06-20T20:41:10.933912Z","shell.execute_reply":"2021-06-20T20:41:14.54088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\nfiles = [test_files_path + '/' + f for f in os.listdir(test_files_path)]","metadata":{"execution":{"iopub.status.busy":"2021-06-20T20:41:14.543924Z","iopub.execute_input":"2021-06-20T20:41:14.54432Z","iopub.status.idle":"2021-06-20T20:41:14.554672Z","shell.execute_reply.started":"2021-06-20T20:41:14.544283Z","shell.execute_reply":"2021-06-20T20:41:14.553569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ntokenizer = AutoTokenizer.from_pretrained('../input/k/lichena/coleridge-ner/tokenizer/', add_prefix_space=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T20:43:26.007393Z","iopub.execute_input":"2021-06-20T20:43:26.007769Z","iopub.status.idle":"2021-06-20T20:43:26.129407Z","shell.execute_reply.started":"2021-06-20T20:43:26.007739Z","shell.execute_reply":"2021-06-20T20:43:26.128578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z()0-9]+', ' ', str(txt))\ndef clean_label(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).lower()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.441689Z","iopub.status.idle":"2021-06-20T16:47:08.442641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndata = df.groupby(by='Id')['dataset_label'].apply('|'.join)\nid_input = data.index.values\nlabels_input = data.values\nlabel_set = df['dataset_label'].unique().tolist()\nlabel_set = [clean_text(label) for label in label_set]","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.444057Z","iopub.status.idle":"2021-06-20T16:47:08.445007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunk_text(full_text, length=250, overlap=25):\n    full_text = full_text.split()\n    text_len = len(full_text)\n    results = []\n    i = 0\n    while i < text_len:\n        results.append(' '.join(full_text[i:i+length]))\n        i = i + length - overlap\n    return results","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.446339Z","iopub.status.idle":"2021-06-20T16:47:08.447283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode(filename, min_length=5):\n    txts = []\n    ids = []\n    naive_results = []\n    with open(filename, 'r') as f:\n        json_decode = json.load(f)\n        full_text = ''\n        for data in json_decode: # for each section of the document\n            full_text += ' ' + data['section_title'] + ' ' + data['text']\n        full_text = clean_text(full_text)\n        for label in label_set:\n            if label in full_text and label not in naive_results:\n                naive_results.append(label.lower())\n        chunks = chunk_text(full_text)\n        for chunk in chunks:\n            txts.append(chunk)\n            ids.append(os.path.basename(filename)[:-5])\n        \n    return txts, ids, os.path.basename(filename)[:-5], naive_results","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.448678Z","iopub.status.idle":"2021-06-20T16:47:08.449585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = {}\ntexts = []\nids = []\nfor f in tqdm(files):\n    txts, id_list, _id, naive_results = encode(f)\n    texts += txts\n    ids += id_list\n#     results[_id] = naive_results\n    results[_id] = []","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.450966Z","iopub.status.idle":"2021-06-20T16:47:08.451856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ColeridgeDataset(Dataset):\n    def __init__(self, texts, ids):\n        self.texts = texts\n        self.ids = ids\n        \n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, idx):\n        return self.texts[idx], self.ids[idx]\n\ndef collate_fn(batch):\n    texts = [item[0] for item in batch]\n    ids = [item[1] for item in batch]\n    encoding = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=512)\n    return encoding.input_ids, encoding.attention_mask, ids","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.453206Z","iopub.status.idle":"2021-06-20T16:47:08.454083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = ColeridgeDataset(texts, ids)\ndataloader = DataLoader(dataset, batch_size=128, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.455484Z","iopub.status.idle":"2021-06-20T16:47:08.456331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_titles(input_ids, pred, score, _id):\n    titles = []\n    scores = []\n    ids = []\n    for idx in range(input_ids.shape[0]):\n        is_title = False\n        tmp_toks = []\n        tmp_scores = []\n        for row in range(input_ids.shape[1]):\n            if pred[idx][row] > 0:\n                if is_title and pred[idx][row] == 1:\n                    titles.append(tokenizer.decode(tmp_toks).strip())\n                    scores.append((sum(tmp_scores) / len(tmp_scores)).item())\n                    ids.append(_id[idx])\n                    tmp_toks = []\n                    tmp_scores = []\n                tmp_toks.append(input_ids[idx][row])\n                tmp_scores.append(score[idx][row])\n                is_title = True\n            elif is_title and (pred[idx][row] == 0 or row == input_ids.shape[1] - 1 or input_ids[idx][row] == 102):\n                is_title = False\n                titles.append(tokenizer.decode(tmp_toks).strip())\n                scores.append((sum(tmp_scores) / len(tmp_scores)).item())\n                ids.append(_id[idx])\n                tmp_toks = []\n                tmp_scores = []\n            elif is_title:\n                tmp_toks.append(input_ids[idx][row])\n                tmp_scores.append(score[idx][row])\n    return titles, scores, ids","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.45772Z","iopub.status.idle":"2021-06-20T16:47:08.45862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_fn(dataloader, model):\n    model.eval()\n    with torch.no_grad():\n        titles = []\n        scores = []\n        ids = []\n        loader = tqdm(dataloader)\n        for batch in loader:\n            input_ids, attention_mask, _id = batch\n            input_ids = input_ids.to(DEVICE)\n            attention_mask = attention_mask.type(torch.uint8).to(DEVICE)\n            emissions = model(input_ids, attention_mask=attention_mask).logits\n            pred = torch.tensor(crf.decode(emissions))\n            score = torch.max(torch.softmax(emissions.permute(0,2,1), dim=1), dim=1).values\n            _titles, _scores, _ids = get_titles(input_ids, pred, score, _id)\n            titles += _titles\n            scores += _scores\n            ids += _ids\n        return titles, scores, ids","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.459979Z","iopub.status.idle":"2021-06-20T16:47:08.460884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load('../input/k/lichena/coleridge-ner/checkpoint.pt', map_location=DEVICE)\nmodel = checkpoint['model']\ncrf = checkpoint['crf']\nmodel.to(DEVICE);\ncrf.to(DEVICE);","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.462167Z","iopub.status.idle":"2021-06-20T16:47:08.463059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles, scores, ids = eval_fn(dataloader, model)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.464206Z","iopub.status.idle":"2021-06-20T16:47:08.46474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lcs(X, Y):\n    m = len(X)\n    n = len(Y)\n \n    # Create a table to store lengths of\n    # longest common suffixes of substrings.\n    # Note that LCSuff[i][j] contains length\n    # of longest common suffix of X[0..i-1] and\n    # Y[0..j-1]. The first row and first\n    # column entries have no logical meaning,\n    # they are used only for simplicity of program\n    LCSuff = [[0 for i in range(n + 1)]\n                 for j in range(m + 1)]\n \n    # To store length of the\n    # longest common substring\n    length = 0\n \n    # To store the index of the cell\n    # which contains the maximum value.\n    # This cell's index helps in building\n    # up the longest common substring\n    # from right to left.\n    row, col = 0, 0\n \n    # Following steps build LCSuff[m+1][n+1]\n    # in bottom up fashion.\n    for i in range(m + 1):\n        for j in range(n + 1):\n            if i == 0 or j == 0:\n                LCSuff[i][j] = 0\n            elif X[i - 1] == Y[j - 1]:\n                LCSuff[i][j] = LCSuff[i - 1][j - 1] + 1\n                if length < LCSuff[i][j]:\n                    length = LCSuff[i][j]\n                    row = i\n                    col = j\n            else:\n                LCSuff[i][j] = 0\n \n    # if true, then no common substring exists\n    if length == 0:\n        return\n \n    # allocate space for the longest\n    # common substring\n    resultStr = ['0'] * length\n \n    # traverse up diagonally form the\n    # (row, col) cell until LCSuff[row][col] != 0\n    while LCSuff[row][col] != 0:\n        length -= 1\n        resultStr[length] = X[row - 1] # or Y[col-1]\n \n        # move diagonally up to previous cell\n        row -= 1\n        col -= 1\n \n    # required longest common substring\n    return ''.join(resultStr)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T16:47:08.465656Z","iopub.status.idle":"2021-06-20T16:47:08.46623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stops = stopwords.words('english')\nstops.append('[sep]')\nstops.append('[PAD]')\nstops.append('[pad]')\nstops.append('pad')\nstops.append('PAD')\n\n\ndef clean_front(split):\n    for idx, e in enumerate(split):\n        if e not in stops:\n            return split[idx:]\n        \ndef clean_back(split):\n    for i in reversed(range(len(split))):\n        if split[i] not in stops:\n            return split[:i+1]\n        \ndef clean_result(title):\n    title = title.replace('<s>', '')\n    title = title.replace('</s>', '')\n    split = title.split()\n    if split:\n        if len(split[-1]) <= 2:\n            split = split[:-1]\n    if split:\n        split = clean_front(split)\n    if split:\n        split = clean_back(split)\n    if split:\n        title = ' '.join(split)\n        return title","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:16:58.272839Z","iopub.execute_input":"2021-06-14T17:16:58.273267Z","iopub.status.idle":"2021-06-14T17:16:58.284586Z","shell.execute_reply.started":"2021-06-14T17:16:58.273231Z","shell.execute_reply":"2021-06-14T17:16:58.283511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = checkpoint['thresh']\nprint(checkpoint['thresh'])\nprint(checkpoint['epoch'])\n# threshold = 0.6\nfor idx, _id in enumerate(tqdm(ids)):\n    if _id not in results:\n        results[_id] = []\n    else:\n        title = clean_label(titles[idx])\n        if title and len(title) > 2 and scores[idx] > threshold and ' ' in title:\n            cleaned = clean_result(title)\n            if cleaned and cleaned not in results[_id]:\n                is_new = True\n                for idx, element in enumerate(results[_id]):\n                    if element in cleaned:\n                        is_new = False\n                        results[_id][idx] = cleaned\n                    elif cleaned in element:\n                        is_new = False\n                        results[_id][idx] = element\n                if is_new:\n                    results[_id].append(cleaned)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:17:06.740107Z","iopub.execute_input":"2021-06-14T17:17:06.740522Z","iopub.status.idle":"2021-06-14T17:17:06.755293Z","shell.execute_reply.started":"2021-06-14T17:17:06.740485Z","shell.execute_reply":"2021-06-14T17:17:06.753959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv \n\n# name of csv file \nfilename = \"submission.csv\"\n    \n# writing to csv file \nwith open(filename, 'w') as csvfile: \n    # creating a csv writer object \n    csvwriter = csv.writer(csvfile) \n    \n    # writing the fields \"\n    csvwriter.writerow([\"Id\", \"PredictionString\"]) \n\n    for (k,v) in results.items():\n        if len(v) > 0:\n            # writing the data rows \n            csvwriter.writerow([k, '|'.join(v)])\n        else:\n            csvwriter.writerow([k, ''])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:17:08.795817Z","iopub.execute_input":"2021-06-14T17:17:08.796219Z","iopub.status.idle":"2021-06-14T17:17:08.803746Z","shell.execute_reply.started":"2021-06-14T17:17:08.796179Z","shell.execute_reply":"2021-06-14T17:17:08.802476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results","metadata":{"execution":{"iopub.status.busy":"2021-06-14T17:17:09.121424Z","iopub.execute_input":"2021-06-14T17:17:09.121816Z","iopub.status.idle":"2021-06-14T17:17:09.128532Z","shell.execute_reply.started":"2021-06-14T17:17:09.121782Z","shell.execute_reply":"2021-06-14T17:17:09.127412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# titles\n# results\n# run 51 was pretty good","metadata":{"execution":{"iopub.status.busy":"2021-06-14T16:44:01.205386Z","iopub.execute_input":"2021-06-14T16:44:01.205747Z","iopub.status.idle":"2021-06-14T16:44:01.216551Z","shell.execute_reply.started":"2021-06-14T16:44:01.205708Z","shell.execute_reply":"2021-06-14T16:44:01.215389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}