{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\nfrom IPython.display import clear_output\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T13:19:04.268799Z","iopub.execute_input":"2021-06-17T13:19:04.269232Z","iopub.status.idle":"2021-06-17T13:19:37.576448Z","shell.execute_reply.started":"2021-06-17T13:19:04.269145Z","shell.execute_reply":"2021-06-17T13:19:37.575127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade fsspec","metadata":{"execution":{"iopub.status.busy":"2021-06-17T13:20:30.721807Z","iopub.execute_input":"2021-06-17T13:20:30.722195Z","iopub.status.idle":"2021-06-17T13:20:38.223452Z","shell.execute_reply.started":"2021-06-17T13:20:30.722162Z","shell.execute_reply":"2021-06-17T13:20:38.22213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\n\nimport os\nimport re\nimport json\nimport time\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.autonotebook import tqdm\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\nimport warnings\nwarnings.filterwarnings(\"ignore\", 'This pattern has match groups')\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-17T13:20:42.38606Z","iopub.execute_input":"2021-06-17T13:20:42.386446Z","iopub.status.idle":"2021-06-17T13:20:42.431847Z","shell.execute_reply.started":"2021-06-17T13:20:42.386413Z","shell.execute_reply":"2021-06-17T13:20:42.429514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILES_PATH = \"../input/coleridgeinitiative-show-us-the-data/train\"\nTRAIN_PATH = \"../input/coleridgeinitiative-show-us-the-data/train.csv\"\ntrain = pd.read_csv(TRAIN_PATH)\n\nTEST_FILES_PATH = \"../input/coleridgeinitiative-show-us-the-data/test\"\n\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    \nEXTERNAL_DATASET_PATH = '../input/filtered-bigger-govt-dataset/ExtraLabelsCleaned.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:44:10.704277Z","iopub.execute_input":"2021-06-05T11:44:10.704635Z","iopub.status.idle":"2021-06-05T11:44:10.872902Z","shell.execute_reply.started":"2021-06-05T11:44:10.704603Z","shell.execute_reply":"2021-06-05T11:44:10.871843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"included_word_dict = {}\nunincluded_word_dict = {}\n\nfor id in tqdm(train['Id'].unique()):\n    labels = train[train.Id == id][['dataset_title', 'dataset_label', 'cleaned_label']]\n    for col in ['dataset_title', 'dataset_label', 'cleaned_label']:\n        labels[col] = labels[col].apply(text_cleaning)\n    labels = np.unique(labels.values.ravel())\n    sentences = set([clean_paper_sentence(sentence) for section in papers[id]\n                     for sentence in section['text'].split('.')\n                    ])\n#     print(labels)\n    for sentence in sentences:\n        flag = 0\n        for label in labels:\n            if label in sentence:\n                flag = 1\n                break\n        if flag == 1:\n            for word in sentence.split():\n                if word in included_word_dict.keys():\n                    included_word_dict[word] += 1\n                else:\n                    included_word_dict[word] = 1\n        else:\n            for word in sentence.split():\n                if not word in unincluded_word_dict.keys():\n                    unincluded_word_dict[word] = 1\n                else:\n                    unincluded_word_dict[word] += 1\n                    \n                        ","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:48:19.715863Z","iopub.execute_input":"2021-06-05T11:48:19.716271Z","iopub.status.idle":"2021-06-05T11:52:47.482357Z","shell.execute_reply.started":"2021-06-05T11:48:19.716233Z","shell.execute_reply":"2021-06-05T11:52:47.480936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = {}\n\nfor word in included_word_dict.keys():\n    if word in unincluded_word_dict.keys():\n        res[word] = included_word_dict[word] / (unincluded_word_dict[word] + included_word_dict[word])\n    else:\n        res[word] = 1\n        \n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:52:53.304931Z","iopub.execute_input":"2021-06-05T11:52:53.305329Z","iopub.status.idle":"2021-06-05T11:52:53.322334Z","shell.execute_reply.started":"2021-06-05T11:52:53.305286Z","shell.execute_reply":"2021-06-05T11:52:53.321344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_res = sorted(res.items(), reverse=True, key=lambda x: x[1])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:53:23.469914Z","iopub.execute_input":"2021-06-05T11:53:23.470319Z","iopub.status.idle":"2021-06-05T11:53:23.479027Z","shell.execute_reply.started":"2021-06-05T11:53:23.470287Z","shell.execute_reply":"2021-06-05T11:53:23.478142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"included_word_dict['unactuated']","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:56:13.167327Z","iopub.execute_input":"2021-06-05T11:56:13.167818Z","iopub.status.idle":"2021-06-05T11:56:13.173027Z","shell.execute_reply.started":"2021-06-05T11:56:13.167785Z","shell.execute_reply":"2021-06-05T11:56:13.172244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = sorted(included_word_dict.items(), reverse=True, key=lambda x: x[1])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:24:42.629811Z","iopub.execute_input":"2021-06-05T12:24:42.6303Z","iopub.status.idle":"2021-06-05T12:24:42.640533Z","shell.execute_reply.started":"2021-06-05T12:24:42.630256Z","shell.execute_reply":"2021-06-05T12:24:42.639839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[x for x in result if len(x[0]) >= 4]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:25:38.664715Z","iopub.execute_input":"2021-06-05T12:25:38.665369Z","iopub.status.idle":"2021-06-05T12:25:38.736816Z","shell.execute_reply.started":"2021-06-05T12:25:38.665334Z","shell.execute_reply":"2021-06-05T12:25:38.736121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted(included_word_dict.items(), reverse=True, key=lambda x: x[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in tqdm(train['Id'].unique()):\n    with open(f'{TRAIN_FILES_PATH}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:44:52.909817Z","iopub.execute_input":"2021-06-05T11:44:52.910436Z","iopub.status.idle":"2021-06-05T11:46:02.151606Z","shell.execute_reply.started":"2021-06-05T11:44:52.910386Z","shell.execute_reply":"2021-06-05T11:46:02.150472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\ndef read_json_pub(filename, train_data_path=TRAIN_FILES_PATH, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:44:37.692784Z","iopub.execute_input":"2021-06-05T11:44:37.693207Z","iopub.status.idle":"2021-06-05T11:44:37.705344Z","shell.execute_reply.started":"2021-06-05T11:44:37.693171Z","shell.execute_reply":"2021-06-05T11:44:37.704456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#この段階で全てクリーニングしています\nall_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(text_cleaning(label_1))\n    all_labels.add(text_cleaning(label_2))\n    all_labels.add(text_cleaning(label_3))\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:33.403879Z","iopub.execute_input":"2021-06-05T11:46:33.404226Z","iopub.status.idle":"2021-06-05T11:46:34.123558Z","shell.execute_reply.started":"2021-06-05T11:46:33.404196Z","shell.execute_reply":"2021-06-05T11:46:34.122651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#この段階で全てクリーニングしています\nexternal_dataset = pd.read_csv(EXTERNAL_DATASET_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:35.4337Z","iopub.execute_input":"2021-06-05T11:46:35.434076Z","iopub.status.idle":"2021-06-05T11:46:35.447627Z","shell.execute_reply.started":"2021-06-05T11:46:35.434044Z","shell.execute_reply":"2021-06-05T11:46:35.446554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"external_dataset2 = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv').drop_duplicates(subset='title')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:36.108933Z","iopub.execute_input":"2021-06-05T11:46:36.10928Z","iopub.status.idle":"2021-06-05T11:46:36.135045Z","shell.execute_reply.started":"2021-06-05T11:46:36.109249Z","shell.execute_reply":"2021-06-05T11:46:36.134113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label in external_dataset.Label:\n    all_labels.add(text_cleaning(label))\n\nall_labels = set(all_labels)\nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:36.80394Z","iopub.execute_input":"2021-06-05T11:46:36.804344Z","iopub.status.idle":"2021-06-05T11:46:36.810681Z","shell.execute_reply.started":"2021-06-05T11:46:36.804307Z","shell.execute_reply":"2021-06-05T11:46:36.810004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n\nPREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:39.178646Z","iopub.execute_input":"2021-06-05T11:46:39.179149Z","iopub.status.idle":"2021-06-05T11:46:39.182784Z","shell.execute_reply.started":"2021-06-05T11:46:39.179116Z","shell.execute_reply":"2021-06-05T11:46:39.181833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import RobertaTokenizer, RobertaForMaskedLM, pipeline\n\ntokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\", use_fast=True)\nmodel = RobertaForMaskedLM.from_pretrained(\"roberta-base\")\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:46:41.111429Z","iopub.execute_input":"2021-06-05T11:46:41.111976Z","iopub.status.idle":"2021-06-05T11:47:41.900337Z","shell.execute_reply.started":"2021-06-05T11:46:41.11192Z","shell.execute_reply":"2021-06-05T11:47:41.899368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mlm = pipeline(\n    'fill-mask', \n    model=model,\n    tokenizer=tokenizer,\n    device=0 if torch.cuda.is_available() else -1\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T11:47:44.67253Z","iopub.execute_input":"2021-06-05T11:47:44.672918Z","iopub.status.idle":"2021-06-05T11:47:44.678087Z","shell.execute_reply.started":"2021-06-05T11:47:44.672882Z","shell.execute_reply":"2021-06-05T11:47:44.677111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'at', 'for', 'data', 'dataset', 'datasets'}#atとdetasetsを入れた\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):#文頭は必ず大文字だから\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    #追加　phraseからconnection_tokenを除去する\n                    while sentence[phrase_start].lower() in connection_tokens:\n                        phrase_start += 1\n                    while sentence[phrase_end].lower() in connection_tokens:\n                        phrase_end -= 1\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            #追加　phraseからconnection_tokenを除去する\n            while sentence[phrase_start].lower() in connection_tokens:\n                phrase_start += 1\n            while sentence[phrase_end].lower() in connection_tokens:\n                phrase_end -= 1\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:41:13.593587Z","iopub.execute_input":"2021-06-05T12:41:13.59401Z","iopub.status.idle":"2021-06-05T12:41:13.612883Z","shell.execute_reply.started":"2021-06-05T12:41:13.593975Z","shell.execute_reply":"2021-06-05T12:41:13.611624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = mlm.tokenizer.mask_token\nall_data = []\n\n#id被りの除去\n# train_ids = set()\n# for id in train['Id']:\n#     train_ids.add(id)\n    \n#for paper_id in tqdm(train['Id']):#これだと同じ論文から同じsentenceが複数できてしまう\nfor paper_id in tqdm(train['Id'].unique()):\n    # load paper\n    paper = papers[paper_id]\n\n    # extract sentences\n    sentences = set([clean_paper_sentence(sentence) for section in paper \n                     for sentence in section['text'].split('.')\n                    ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    #↓dataかstudyを持っているsentenceのみ残しているけどいる？\n#     sentences = [\n#         sentence for sentence in sentences if any(\n#             word in sentence.lower() for word in ['data','students','database', 'study']\n#         )]\n    sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n\n    # mask\n    #train_data = []\n    for sentence in sentences:\n        for phrase_start, phrase_end in find_mask_candidates(sentence):\n            if ' '.join(sentence[phrase_start:phrase_end+1]).lower() in all_labels:\n                dt_point = sentence[:phrase_start] + [DATASET_SYMBOL] + sentence[phrase_end+1:]\n            else:\n                dt_point = sentence[:phrase_start] + [NONDATA_SYMBOL] + sentence[phrase_end+1:]\n            all_data.append([' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]), paper_id]) # [masked text, phrase, id]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:42:36.427327Z","iopub.execute_input":"2021-06-05T12:42:36.427717Z","iopub.status.idle":"2021-06-05T12:46:06.675542Z","shell.execute_reply.started":"2021-06-05T12:42:36.427684Z","shell.execute_reply":"2021-06-05T12:46:06.674849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_df = pd.DataFrame(all_data, columns=['text','phrase','id'] )","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:48:37.368226Z","iopub.execute_input":"2021-06-05T12:48:37.36865Z","iopub.status.idle":"2021-06-05T12:48:38.287605Z","shell.execute_reply.started":"2021-06-05T12:48:37.368615Z","shell.execute_reply":"2021-06-05T12:48:38.286648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('./data', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:51:50.521926Z","iopub.execute_input":"2021-06-05T12:51:50.522426Z","iopub.status.idle":"2021-06-05T12:51:50.526796Z","shell.execute_reply.started":"2021-06-05T12:51:50.522392Z","shell.execute_reply":"2021-06-05T12:51:50.5257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_df.to_csv('./data/all_data.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T12:52:14.841388Z","iopub.execute_input":"2021-06-05T12:52:14.842014Z","iopub.status.idle":"2021-06-05T12:52:30.68379Z","shell.execute_reply.started":"2021-06-05T12:52:14.841964Z","shell.execute_reply":"2021-06-05T12:52:30.682899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}