{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Basic approach using regex and an electra model trained on squad2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"from pathlib import Path\n\ndata_dir = Path('/kaggle/input/coleridgeinitiative-show-us-the-data')\ntest_dir = data_dir/'test'\n\nimport json\n\ndef get_document_text(filename, test=False):\n    if test:\n        filepath = test_dir/(filename+'.json')\n    else:\n        filepath = train_dir/(filename+'.json')\n        \n    with open(filepath, 'r') as f:\n        return \" \".join([_['text'] for _ in json.load(f)])\n    return \"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:40.495261Z","iopub.execute_input":"2021-06-22T19:02:40.495678Z","iopub.status.idle":"2021-06-22T19:02:40.506228Z","shell.execute_reply.started":"2021-06-22T19:02:40.495579Z","shell.execute_reply":"2021-06-22T19:02:40.505228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nalphabets= \"([A-Za-z])\"\nprefixes = re.compile(\"(Mr|St|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|Mt)[.]\")\nsuffixes = \"(Inc|Ltd|Jr|Sr|Co)\"\nstarters = \"(Mr|Mrs|Ms|Dr|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)\"\nacronyms = \"([A-Z][.][A-Z][.](?:[A-Z][.])?)\"\nwebsites = re.compile(\"[.](co|net|org|io|gov|edu|us)\")\netal = re.compile(r\"(\\bet al)[.]\")\nurls = re.compile(\"(www)[.]\")\ndigits =  re.compile(\"[.]([0-9])\")\n\ndef split_into_sentences(text):\n    text = \" \" + text + \"  \"\n    text = text.replace(\"\\n\",\" \")\n    text = prefixes.sub(\"\\\\1<prd>\",text)\n    text = websites.sub(\"<prd>\\\\1\",text)\n    text = urls.sub(\"\\\\1<prd>\",text)\n    if \"Ph.D\" in text: text = text.replace(\"Ph.D.\",\"Ph<prd>D<prd>\")\n    text = re.sub(\"\\s\" + alphabets + \"[.] \",\" \\\\1<prd> \",text)\n    text = re.sub(acronyms+\" \"+starters,\"\\\\1<stop> \\\\2\",text)\n    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\\\\3<prd>\",text)\n    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\",text)\n    text = re.sub(\" \"+suffixes+\"[.] \"+starters,\" \\\\1<stop> \\\\2\",text)\n    text = re.sub(\" \"+suffixes+\"[.]\",\" \\\\1<prd>\",text)\n    text = re.sub(\" \" + alphabets + \"[.]\",\" \\\\1<prd>\",text)\n    text = etal.sub(\"\\\\1<prd>\", text)\n    text = digits.sub(\"<prd>\\\\1\",text)\n    if \"”\" in text: text = text.replace(\".”\",\"”.\")\n    if \"\\\"\" in text: text = text.replace(\".\\\"\",\"\\\".\")\n    if \"!\" in text: text = text.replace(\"!\\\"\",\"\\\"!\")\n    if \"?\" in text: text = text.replace(\"?\\\"\",\"\\\"?\")\n    text = text.replace(\".\",\".<stop>\")\n    text = text.replace(\"?\",\"?<stop>\")\n    text = text.replace(\"!\",\"!<stop>\")\n    text = text.replace(\"<prd>\",\".\")\n    sentences = text.split(\"<stop>\")\n    if sentences[-1] == '':\n        sentences = sentences[:-1]\n    sentences = [s.strip() for s in sentences]\n    return sentences\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower().strip())\n\ndef shorten_sentences(sentences, max_length=80, overlap=25):\n    \"\"\"\n    If a sentence is longer than `max_length`, break it into chunks of \n    length `max_length` with an overlap of length `overlap`.\n    \n    e.g. if the sentence has 50 tokens, max_length is 20, and overlap is 10.\n    Then the first sentence will be token_i where i in [0,20)\n    Second sentence will be token_i in [10,30).\n    Third sentence [20, 40)\n    Fourth [30, 50)\n    Fifth [40, 60)\n    \"\"\"\n    shortened_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        num_words = len(words)\n        if num_words > max_length:\n            for start_index in range(0, num_words, max_length - overlap):\n                shortened_sentences.append(' '.join(words[start_index:start_index+max_length]))\n        else:\n            shortened_sentences.append(sentence)\n    return shortened_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:40.507934Z","iopub.execute_input":"2021-06-22T19:02:40.508454Z","iopub.status.idle":"2021-06-22T19:02:40.528030Z","shell.execute_reply.started":"2021-06-22T19:02:40.508383Z","shell.execute_reply":"2021-06-22T19:02:40.526952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# regex_match_sentences = [] # list of list of sentences found through regex, each element has a list of sentences, one element per id\n# ids = []\n\n\n# keywords = ['longitudinal', \"national\", \"data\", \"model\", \"questionnaire\", \"from\",  \"according\", \"\\buse\\b\", \"\\busing\", \"participants\", \"cohort\", \"studies\", \"study\", \"survey\", \"sample\", \"results\"]\n# keyword_pattern = re.compile(r\"|\".join(keywords))\n\n# for filename in test_dir.iterdir():\n#     file_id = filename.stem\n#     ids.append(file_id)\n#     document_text = get_document_text(file_id, test=True)\n#     sentences = split_into_sentences(document_text)\n    \n#     file_sentences = [sentence for sentence in sentences if keyword_pattern.search(sentence.lower())]            \n    \n#     regex_match_sentences.append(file_sentences)\n\nimport fasttext\n\nft_model = fasttext.load_model(\"../input/coleridge-fasttext-classification/fasttext_model_coleridge.bin\")\n\nfound_sentences = [] #sentences to be ran through NER model later, one element for each file id\nids = []\n\nfor filename in test_dir.iterdir():\n    file_id = filename.stem\n    ids.append(file_id)\n    document_text = get_document_text(file_id, test=True)\n    sentences = split_into_sentences(document_text)\n    \n    file_sentences = []\n    for sentence in sentences:\n        result = ft_model.predict(sentence.lower())\n        if \"has_dataset\" in result[0][0]:\n            file_sentences.append(sentence)\n    \n    found_sentences.append(file_sentences)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:40.529851Z","iopub.execute_input":"2021-06-22T19:02:40.530228Z","iopub.status.idle":"2021-06-22T19:02:56.423123Z","shell.execute_reply.started":"2021-06-22T19:02:40.530193Z","shell.execute_reply":"2021-06-22T19:02:56.421470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig\n\n# classifier_approved_sentences = []\n\n# batch_size = 8\n# tokenizer_path = \"../input/roberta-tokenizer\"\n# text_classifier_model_path = '../input/coleridge-text-class-robertalarge/output-roberta-large'\n\n# config = AutoConfig.from_pretrained(text_classifier_model_path)\n# tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, config=config)\n# model = AutoModelForSequenceClassification.from_pretrained(text_classifier_model_path)\n\n# model.to(\"cuda\")\n# model.eval()\n\n# with torch.no_grad():\n\n#     for sentences in found_sentences:\n#         file_sentences = []\n#         for i in range(0, len(sentences), batch_size):\n#             batch = sentences[i:i+batch_size]\n#             inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=256)\n#             input_ids = inputs[\"input_ids\"].to(\"cuda\")\n#             attention_mask = inputs[\"attention_mask\"].to(\"cuda\")\n#             outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n#             logits = outputs.logits\n# #             probas = logits.softmax(-1).tolist()\n#             predictions = logits.argmax(-1).tolist()\n#             for offset, prediction in enumerate(predictions):\n#                 if prediction == 1:\n#                     file_sentences.append(sentences[i+offset])\n#         classifier_approved_sentences.append(file_sentences)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:56.424706Z","iopub.execute_input":"2021-06-22T19:02:56.425064Z","iopub.status.idle":"2021-06-22T19:02:56.429776Z","shell.execute_reply.started":"2021-06-22T19:02:56.425026Z","shell.execute_reply":"2021-06-22T19:02:56.428835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndel ft_model\n# del found_sentences\n# del tokenizer\n# del config\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:56.431405Z","iopub.execute_input":"2021-06-22T19:02:56.431785Z","iopub.status.idle":"2021-06-22T19:02:56.570339Z","shell.execute_reply.started":"2021-06-22T19:02:56.431747Z","shell.execute_reply":"2021-06-22T19:02:56.569423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import pipeline\n\nelectra_model_path = \"../input/electra-squad2/electra_squad2\"\n\nmodel = pipeline(\"question-answering\", model=electra_model_path, tokenizer=electra_model_path, device=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:17.650913Z","iopub.execute_input":"2021-06-22T19:12:17.651295Z","iopub.status.idle":"2021-06-22T19:12:27.777164Z","shell.execute_reply.started":"2021-06-22T19:12:17.651262Z","shell.execute_reply":"2021-06-22T19:12:27.776305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_predictions = []\n\nquestion_keywords = [\"survey\", \"study\", \"questionnaire\"]\nquestion_template = \"What is the name of the {keyword} being used?\"\n\n\nfor sentences in found_sentences:\n    file_predictions = []\n    unq_preds = set()\n    for sentence in sentences:\n        output = \"\"\n        for keyword in question_keywords:\n            if re.search(keyword, sentence.lower()):\n                output = model(question=question_template.format(keyword=keyword), context=sentence)\n                break\n        if output == \"\":\n            output = model(question=question_template.format(keyword=\"data source\"), context=sentence)     \n        if output[\"score\"] > 0.75:\n            answer = output[\"answer\"]\n            if clean_text(answer) not in unq_preds:\n                file_predictions.append(answer)\n                unq_preds.add(clean_text(answer))\n        \n    all_predictions.append(file_predictions)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:27.779556Z","iopub.execute_input":"2021-06-22T19:12:27.779819Z","iopub.status.idle":"2021-06-22T19:12:37.822413Z","shell.execute_reply.started":"2021-06-22T19:12:27.779793Z","shell.execute_reply":"2021-06-22T19:12:37.821461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:06:42.540127Z","iopub.execute_input":"2021-06-22T19:06:42.540497Z","iopub.status.idle":"2021-06-22T19:06:42.781583Z","shell.execute_reply.started":"2021-06-22T19:06:42.540462Z","shell.execute_reply":"2021-06-22T19:06:42.780473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_strings = []\n\nfor file_predictions in all_predictions:\n    temp_predictions = []\n    for pred in file_predictions:\n        words = pred.split()\n        if len(words) == 1 and words[0].isupper():\n            temp_predictions.append(clean_text(pred))\n        else:\n            try:\n                if words[0][0].islower() and words[1][0].islower():\n                    continue\n            except IndexError:\n                pass\n            if \"et al.\" in pred:\n                continue\n            if pred.islower():\n                continue\n            temp_predictions.append(clean_text(pred))\n    \n    prediction_strings.append(\"|\".join(temp_predictions))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:37.823993Z","iopub.execute_input":"2021-06-22T19:12:37.824312Z","iopub.status.idle":"2021-06-22T19:12:37.831908Z","shell.execute_reply.started":"2021-06-22T19:12:37.824282Z","shell.execute_reply":"2021-06-22T19:12:37.830969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prediction_strings = [\"|\".join(x) for x in all_predictions]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:37.833512Z","iopub.execute_input":"2021-06-22T19:12:37.834156Z","iopub.status.idle":"2021-06-22T19:12:37.846007Z","shell.execute_reply.started":"2021-06-22T19:12:37.834117Z","shell.execute_reply":"2021-06-22T19:12:37.845107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nsubmission_df = pd.DataFrame(data={\"Id\":ids, \"PredictionString\":prediction_strings})\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:37.847485Z","iopub.execute_input":"2021-06-22T19:12:37.848216Z","iopub.status.idle":"2021-06-22T19:12:37.867221Z","shell.execute_reply.started":"2021-06-22T19:12:37.848173Z","shell.execute_reply":"2021-06-22T19:12:37.866486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}