{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Start writing code here...\nimport numpy as np \nimport pandas as pd\n\npd.options.mode.chained_assignment = None \n\nimport os, re\nimport json\nimport matplotlib.pyplot as plt\n  \nfrom IPython.display import display, clear_output\n    \nfrom tqdm import tqdm\nimport string\n\nfrom sklearn.model_selection import train_test_split\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:13:00.829938Z","iopub.execute_input":"2021-06-13T20:13:00.830349Z","iopub.status.idle":"2021-06-13T20:13:01.837273Z","shell.execute_reply.started":"2021-06-13T20:13:00.830261Z","shell.execute_reply":"2021-06-13T20:13:01.836196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation = pd.read_csv(\"../input/colerigde-processed-text/validation.csv\",index_col=0)\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:08.849983Z","iopub.execute_input":"2021-06-13T20:20:08.850416Z","iopub.status.idle":"2021-06-13T20:20:08.930396Z","shell.execute_reply.started":"2021-06-13T20:20:08.850382Z","shell.execute_reply":"2021-06-13T20:20:08.929107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\n\ndevice_lib.list_local_devices()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:13:02.030423Z","iopub.execute_input":"2021-06-13T20:13:02.030846Z","iopub.status.idle":"2021-06-13T20:13:10.210338Z","shell.execute_reply.started":"2021-06-13T20:13:02.030812Z","shell.execute_reply":"2021-06-13T20:13:10.209138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links \"../input/spacy3\" spacy[cuda110]\n!pip install --no-index --find-links \"../input/spacy3\" en_core_web_trf\n!pip install --no-index --find-links \"../input/spacy3\" spacy_transformers","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:13:10.212323Z","iopub.execute_input":"2021-06-13T20:13:10.212591Z","iopub.status.idle":"2021-06-13T20:14:04.198936Z","shell.execute_reply.started":"2021-06-13T20:13:10.212563Z","shell.execute_reply":"2021-06-13T20:14:04.196520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nfrom spacy.tokens import DocBin","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:14:04.202795Z","iopub.execute_input":"2021-06-13T20:14:04.203572Z","iopub.status.idle":"2021-06-13T20:14:11.134097Z","shell.execute_reply.started":"2021-06-13T20:14:04.203524Z","shell.execute_reply":"2021-06-13T20:14:11.132877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(spacy.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:14:11.135850Z","iopub.execute_input":"2021-06-13T20:14:11.136325Z","iopub.status.idle":"2021-06-13T20:14:11.147117Z","shell.execute_reply.started":"2021-06-13T20:14:11.136279Z","shell.execute_reply":"2021-06-13T20:14:11.146037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model\ntry:\n    spacy.require_gpu()\nexcept:\n    print(\"GPU not found\")\n    \nnlp = spacy.load(R\"../input/coledridge-challenge-models/model-best\")","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:14:11.148656Z","iopub.execute_input":"2021-06-13T20:14:11.149362Z","iopub.status.idle":"2021-06-13T20:14:27.436690Z","shell.execute_reply.started":"2021-06-13T20:14:11.149313Z","shell.execute_reply":"2021-06-13T20:14:27.435419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing the model\ndoc = nlp('The supply of PCR reagents, trained lab personnel and the availability of laboratories with sufficient biocontainment levels are major challenges of SARS-CoV-2 detection in developing countries, such as Indonesia (Younes et al., 2020) . Therefore, it is not surprising that the tested people per week is still lower than the World Health Organization (WHO) standard (World Health Organization, 2020b) . Recently, SARS-CoV-2 with the D614G mutation became the most frequently detected globally, including South East Asia region (Korber et al., 2020; Nguyen et al., 2020) . Interestingly, SARS-CoV-2 with the G614 variant had significantly higher infectious titers than the original D614 virus, and COVID-19 patients with the G614 variant had a higher viral load than patients without the mutation (Korber et al., 2020) . A recent study showed that the SARS-CoV-2 with the G614 variant revealed increased infectivity, competitive fitness, and transmission than the wild-type D614 virus in human airway epithelial cells and hamster (Hou et al., 2020) . However, this mutation was not associated with the severity of COVID-19 (Korber et al., 2020; Nguyen et al., 2020) . Here, we aimed:\\n(1) to report full-length genome sequences of SARS-CoV-2 collected from four COVID-19 patients in the Special Region of Yogyakarta and Central Java provinces, Indonesia;\\n(2) to compare the clade distribution of full-length genome sequences from Indonesia (n = 60) from March to September 2020; and (3) to perform phylogenetic analysis of SARS-CoV-2 complete genomes from different countries, including Indonesia.') # input sample text\nprint(\"Entities\", [(ent.text, ent.label_) for ent in doc.ents])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:12.714236Z","iopub.execute_input":"2021-06-13T20:20:12.714619Z","iopub.status.idle":"2021-06-13T20:20:12.783198Z","shell.execute_reply.started":"2021-06-13T20:20:12.714586Z","shell.execute_reply":"2021-06-13T20:20:12.781795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spacy.displacy.render(doc, style=\"ent\", jupyter=True) # display in Jupyter","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:12.946869Z","iopub.execute_input":"2021-06-13T20:20:12.947321Z","iopub.status.idle":"2021-06-13T20:20:12.955165Z","shell.execute_reply.started":"2021-06-13T20:20:12.947288Z","shell.execute_reply":"2021-06-13T20:20:12.953837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:13.587927Z","iopub.execute_input":"2021-06-13T20:20:13.588343Z","iopub.status.idle":"2021-06-13T20:20:13.605337Z","shell.execute_reply.started":"2021-06-13T20:20:13.588311Z","shell.execute_reply":"2021-06-13T20:20:13.603787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing the model\ndoc = nlp(\"investigated the differences in college access and choice among students of different racialethnic groups using the national educational longitudinal study nels8892 and the beginning postsecondary student longitudinal study bps9092\") # input sample text\nprint(\"Entities\", [(ent.text, ent.label_) for ent in doc.ents])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:14.099310Z","iopub.execute_input":"2021-06-13T20:20:14.099682Z","iopub.status.idle":"2021-06-13T20:20:14.150214Z","shell.execute_reply.started":"2021-06-13T20:20:14.099651Z","shell.execute_reply":"2021-06-13T20:20:14.148821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spacy.displacy.render(doc, style=\"ent\", jupyter=True) # display in Jupyter","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:14.531139Z","iopub.execute_input":"2021-06-13T20:20:14.531524Z","iopub.status.idle":"2021-06-13T20:20:14.540883Z","shell.execute_reply.started":"2021-06-13T20:20:14.531491Z","shell.execute_reply":"2021-06-13T20:20:14.539512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:22.700151Z","iopub.execute_input":"2021-06-13T20:20:22.700524Z","iopub.status.idle":"2021-06-13T20:20:22.726347Z","shell.execute_reply.started":"2021-06-13T20:20:22.700493Z","shell.execute_reply":"2021-06-13T20:20:22.725109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Validate model","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:23.390175Z","iopub.execute_input":"2021-06-13T20:20:23.390782Z","iopub.status.idle":"2021-06-13T20:20:23.405485Z","shell.execute_reply.started":"2021-06-13T20:20:23.390674Z","shell.execute_reply":"2021-06-13T20:20:23.404186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_ds_from_text(path):\n    extract_df = pd.read_json(path)\n    \n    dataset = []\n\n    for idx, r in extract_df.iterrows():\n        doc = nlp(r['text'])\n\n        if doc.ents:\n            for i in [ent.text for ent in doc.ents]:\n                i = clean_text(i)\n                if len(i) != 0 and i not in dataset:\n                    dataset.append(str(i).strip())\n\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:29:52.920407Z","iopub.execute_input":"2021-06-13T20:29:52.920766Z","iopub.status.idle":"2021-06-13T20:29:52.928730Z","shell.execute_reply.started":"2021-06-13T20:29:52.920735Z","shell.execute_reply":"2021-06-13T20:29:52.927266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:24.288288Z","iopub.execute_input":"2021-06-13T20:20:24.288734Z","iopub.status.idle":"2021-06-13T20:20:24.302656Z","shell.execute_reply.started":"2021-06-13T20:20:24.288672Z","shell.execute_reply":"2021-06-13T20:20:24.294563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jaccard(\"national education longitudinal study\",\"education longitudinal study\")","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:25:37.737058Z","iopub.execute_input":"2021-06-13T20:25:37.737505Z","iopub.status.idle":"2021-06-13T20:25:37.746565Z","shell.execute_reply.started":"2021-06-13T20:25:37.737473Z","shell.execute_reply":"2021-06-13T20:25:37.745124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, rows in tqdm(validation.iterrows()):\n    doc = nlp(rows['Text'])\n\n    preds = []\n    \n    if doc.ents:\n        for i in [ent.text for ent in doc.ents]:\n            i = clean_text(i)\n            if len(i) != 0:\n                preds.append(str(i).strip())\n    \n    validation.loc[index, 'predictions'] = '|'.join(preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:20:25.700802Z","iopub.execute_input":"2021-06-13T20:20:25.701176Z","iopub.status.idle":"2021-06-13T20:23:49.809892Z","shell.execute_reply.started":"2021-06-13T20:20:25.701130Z","shell.execute_reply":"2021-06-13T20:23:49.808408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation['Label'] = validation['Label'].apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:23:49.812479Z","iopub.execute_input":"2021-06-13T20:23:49.812942Z","iopub.status.idle":"2021-06-13T20:23:49.839661Z","shell.execute_reply.started":"2021-06-13T20:23:49.812895Z","shell.execute_reply":"2021-06-13T20:23:49.838135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation['truth'] = validation.groupby(['Id'])['Label'].transform(lambda x : '|'.join(x))\nvalidation['predictions'] = validation.groupby(['Id'])['predictions'].transform(lambda x : '|'.join(x))\nvalidation = validation.drop_duplicates(\"Id\")","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:23:49.842217Z","iopub.execute_input":"2021-06-13T20:23:49.842958Z","iopub.status.idle":"2021-06-13T20:23:51.399843Z","shell.execute_reply.started":"2021-06-13T20:23:49.842910Z","shell.execute_reply":"2021-06-13T20:23:51.398604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation = validation[['Id', 'truth', 'predictions']]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:23:51.401607Z","iopub.execute_input":"2021-06-13T20:23:51.402051Z","iopub.status.idle":"2021-06-13T20:23:51.409981Z","shell.execute_reply.started":"2021-06-13T20:23:51.401984Z","shell.execute_reply":"2021-06-13T20:23:51.408808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:23:51.934545Z","iopub.execute_input":"2021-06-13T20:23:51.934903Z","iopub.status.idle":"2021-06-13T20:23:51.954603Z","shell.execute_reply.started":"2021-06-13T20:23:51.934872Z","shell.execute_reply":"2021-06-13T20:23:51.953129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TP = 0\n\nFP = 0\nFN = 0\n\nfor index, rows in validation.iterrows():\n    predictions = np.array(str(rows['predictions']).split('|'))\n    ground_truths = np.array(str(rows['truth']).split('|'))\n    \n    predictions = np.unique(predictions)\n    ground_truths = np.unique(ground_truths)\n    \n    predictions = np.sort(predictions)\n\n    for pred in predictions:\n        for truth in ground_truths:\n            if pred in truth:\n                if jaccard(pred,truth) >= 0.5:\n                    TP += 1\n                else:\n                    FP += 1\n            else:\n                FP += 1\n                \n            if truth not in predictions:\n                FN += 1","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:23:58.902615Z","iopub.execute_input":"2021-06-13T20:23:58.903088Z","iopub.status.idle":"2021-06-13T20:23:59.443576Z","shell.execute_reply.started":"2021-06-13T20:23:58.903003Z","shell.execute_reply":"2021-06-13T20:23:59.442388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"True positives: {}\".format(TP))\nprint(\"\\n\")\nprint(\"False positives: {}\".format(FP))\nprint(\"False negatives: {}\".format(FN))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:24:00.111621Z","iopub.execute_input":"2021-06-13T20:24:00.112065Z","iopub.status.idle":"2021-06-13T20:24:00.120881Z","shell.execute_reply.started":"2021-06-13T20:24:00.111985Z","shell.execute_reply":"2021-06-13T20:24:00.119157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beta = 0.5\nprecision = (TP/(TP+FP))\nrecall = (TP/(TP+FN))\n\nfbeta = ((1 + beta**2) * precision * recall) / (beta**2 * precision + recall)\n\nprint(\"F0.5 score: {}\".format(fbeta))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:24:01.330953Z","iopub.execute_input":"2021-06-13T20:24:01.331422Z","iopub.status.idle":"2021-06-13T20:24:01.340127Z","shell.execute_reply.started":"2021-06-13T20:24:01.331391Z","shell.execute_reply":"2021-06-13T20:24:01.338747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create Submissions file","metadata":{}},{"cell_type":"code","source":"example = \"../input/coleridgeinitiative-show-us-the-data/test/2f392438-e215-4169-bebf-21ac4ff253e1.json\"\n\nprint(extract_ds_from_text(example))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:29:55.753969Z","iopub.execute_input":"2021-06-13T20:29:55.754619Z","iopub.status.idle":"2021-06-13T20:30:02.374830Z","shell.execute_reply.started":"2021-06-13T20:29:55.754572Z","shell.execute_reply":"2021-06-13T20:30:02.368705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, rows in submission.iterrows():\n    preds = extract_ds_from_text((test_files_path+\"/\"+rows[\"Id\"]+\".json\"))\n    submission.loc[index, 'PredictionString'] = '|'.join(preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:27:00.949492Z","iopub.execute_input":"2021-06-13T20:27:00.949877Z","iopub.status.idle":"2021-06-13T20:27:11.746283Z","shell.execute_reply.started":"2021-06-13T20:27:00.949843Z","shell.execute_reply":"2021-06-13T20:27:11.745072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:27:11.755440Z","iopub.execute_input":"2021-06-13T20:27:11.763188Z","iopub.status.idle":"2021-06-13T20:27:11.784802Z","shell.execute_reply.started":"2021-06-13T20:27:11.763141Z","shell.execute_reply":"2021-06-13T20:27:11.783707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T20:18:04.054438Z","iopub.execute_input":"2021-06-13T20:18:04.054920Z","iopub.status.idle":"2021-06-13T20:18:04.067032Z","shell.execute_reply.started":"2021-06-13T20:18:04.054872Z","shell.execute_reply":"2021-06-13T20:18:04.065494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}