{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Start writing code here...\nimport numpy as np \nimport pandas as pd\n  \nfrom IPython.display import display, clear_output\n    \nfrom tqdm import tqdm\n\nfrom ast import literal_eval\n\nfrom transformers import AutoTokenizer, AutoModel","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:21.596099Z","iopub.execute_input":"2021-06-13T07:55:21.59645Z","iopub.status.idle":"2021-06-13T07:55:21.601513Z","shell.execute_reply.started":"2021-06-13T07:55:21.596418Z","shell.execute_reply":"2021-06-13T07:55:21.600307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\n\ndevice_lib.list_local_devices()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:22.96771Z","iopub.execute_input":"2021-06-13T07:55:22.968072Z","iopub.status.idle":"2021-06-13T07:55:23.006049Z","shell.execute_reply.started":"2021-06-13T07:55:22.968038Z","shell.execute_reply":"2021-06-13T07:55:23.004886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links \"../input/spacy3\" spacy[cuda110]\n!pip install --no-index --find-links \"../input/spacy3\" en_core_web_trf\n!pip install --no-index --find-links \"../input/spacy3\" spacy_transformers","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:54:50.564204Z","iopub.status.idle":"2021-06-13T07:54:50.565002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nfrom spacy.tokens import DocBin","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:26.312409Z","iopub.execute_input":"2021-06-13T07:55:26.312782Z","iopub.status.idle":"2021-06-13T07:55:26.31675Z","shell.execute_reply.started":"2021-06-13T07:55:26.312744Z","shell.execute_reply":"2021-06-13T07:55:26.315784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(spacy.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:27.063029Z","iopub.execute_input":"2021-06-13T07:55:27.063373Z","iopub.status.idle":"2021-06-13T07:55:27.070204Z","shell.execute_reply.started":"2021-06-13T07:55:27.063341Z","shell.execute_reply":"2021-06-13T07:55:27.066986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%env TRANSFORMERS_OFFLINE=1\n%env HF_DATASETS_OFFLINE=1","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:27.729541Z","iopub.execute_input":"2021-06-13T07:55:27.729938Z","iopub.status.idle":"2021-06-13T07:55:27.742011Z","shell.execute_reply.started":"2021-06-13T07:55:27.729905Z","shell.execute_reply":"2021-06-13T07:55:27.740819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pre-existing spacy model\ntry:\n    spacy.require_gpu()\nexcept:\n    print(\"GPU not found\")\n\nnlp = spacy.load('en_core_web_trf', disable=[\"tagger\",\"parser\",\"attribute_ruler\",\"lemmatizer\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:28.184916Z","iopub.execute_input":"2021-06-13T07:55:28.185228Z","iopub.status.idle":"2021-06-13T07:55:29.280878Z","shell.execute_reply.started":"2021-06-13T07:55:28.1852Z","shell.execute_reply":"2021-06-13T07:55:29.279994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = \"../input/scibert-scivocab-uncased\"\nlocal_path = \"./scibert-scivocab-uncased\"\nmodel = AutoModel.from_pretrained(name)\ntokenizer = AutoTokenizer.from_pretrained(name)\n\n# add the setting (note that you can modify tokenizer.model_max_length on the fly, \n# but frustratingly this change isn't saved as part of the saved config)\ntokenizer.init_kwargs[\"model_max_length\"] = 512\n\n# save\ntokenizer.save_pretrained(local_path)\nmodel.save_pretrained(local_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:29.282399Z","iopub.execute_input":"2021-06-13T07:55:29.282793Z","iopub.status.idle":"2021-06-13T07:55:35.545598Z","shell.execute_reply.started":"2021-06-13T07:55:29.282754Z","shell.execute_reply":"2021-06-13T07:55:35.544677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/colerigde-processed-text/train.csv\",index_col=0)\nvalidation = pd.read_csv(\"../input/colerigde-processed-text/validation.csv\",index_col=0)\n\ntrain = train.reset_index(drop=True)\nvalidation = validation.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:35.828849Z","iopub.execute_input":"2021-06-13T07:55:35.829157Z","iopub.status.idle":"2021-06-13T07:55:36.509907Z","shell.execute_reply.started":"2021-06-13T07:55:35.829126Z","shell.execute_reply":"2021-06-13T07:55:36.509041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['dataset_index'] = train['dataset_index'].apply(lambda x: literal_eval(x))\nvalidation['dataset_index'] = validation['dataset_index'].apply(lambda x: literal_eval(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:36.5113Z","iopub.execute_input":"2021-06-13T07:55:36.511656Z","iopub.status.idle":"2021-06-13T07:55:37.068319Z","shell.execute_reply.started":"2021-06-13T07:55:36.511618Z","shell.execute_reply":"2021-06-13T07:55:37.06735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:37.06998Z","iopub.execute_input":"2021-06-13T07:55:37.070386Z","iopub.status.idle":"2021-06-13T07:55:37.091915Z","shell.execute_reply.started":"2021-06-13T07:55:37.070341Z","shell.execute_reply":"2021-06-13T07:55:37.090763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(validation.shape)\nvalidation.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:37.656798Z","iopub.execute_input":"2021-06-13T07:55:37.657109Z","iopub.status.idle":"2021-06-13T07:55:37.676973Z","shell.execute_reply.started":"2021-06-13T07:55:37.657083Z","shell.execute_reply":"2021-06-13T07:55:37.676024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"dataset_index\"][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:38.384129Z","iopub.execute_input":"2021-06-13T07:55:38.384466Z","iopub.status.idle":"2021-06-13T07:55:38.39216Z","shell.execute_reply.started":"2021-06-13T07:55:38.384434Z","shell.execute_reply":"2021-06-13T07:55:38.389853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATA = []\nVALIDATION_DATA = []\n\nfor index, row in train.iterrows():\n    TRAIN_DATA.append((row[\"Text\"],{\"entities\":row[\"dataset_index\"]}))\n    \nfor index, row in validation.iterrows():\n    VALIDATION_DATA.append((row[\"Text\"],{\"entities\":row[\"dataset_index\"]}))\n    \nprint(TRAIN_DATA[10])\nprint(VALIDATION_DATA[10])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:38.951465Z","iopub.execute_input":"2021-06-13T07:55:38.951863Z","iopub.status.idle":"2021-06-13T07:55:41.948547Z","shell.execute_reply.started":"2021-06-13T07:55:38.951823Z","shell.execute_reply":"2021-06-13T07:55:41.947123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert SpaCy v2 to SpaCy v3 object\n\ndb = DocBin() # create a DocBin object\ni = 1\n\nfor text, annot in tqdm(TRAIN_DATA): # data in previous format\n    doc = nlp.make_doc(text) # create doc object from text\n    ents = []\n    for start, end, label in annot[\"entities\"]: # add character indexes\n        span = doc.char_span(start, end, label=label, alignment_mode=\"contract\")\n        if span is None:\n            i += 1\n        else:\n            ents.append(span)\n    doc.ents = ents # label the text with the ents\n    db.add(doc)\n\nprint(\"Skipped \"+ str(i) +\" entities\")\n    \ndb.to_disk(\"train.spacy\") # save the docbin object","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:55:42.342914Z","iopub.execute_input":"2021-06-13T07:55:42.343209Z","iopub.status.idle":"2021-06-13T07:56:36.448372Z","shell.execute_reply.started":"2021-06-13T07:55:42.343182Z","shell.execute_reply":"2021-06-13T07:56:36.44744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"db = DocBin() # create a DocBin object\ni = 1\n\nfor text, annot in tqdm(VALIDATION_DATA): # data in previous format\n    doc = nlp.make_doc(text) # create doc object from text\n    ents = []\n    for start, end, label in annot[\"entities\"]: # add character indexes\n        span = doc.char_span(start, end, label=label, alignment_mode=\"contract\")\n        if span is None:\n            i += 1\n        else:\n            ents.append(span)\n    doc.ents = ents # label the text with the ents\n    db.add(doc)\n\nprint(\"Skipped \"+ str(i) +\" entities\")\n    \ndb.to_disk(\"validation.spacy\") # save the docbin object","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:56:36.449953Z","iopub.execute_input":"2021-06-13T07:56:36.450275Z","iopub.status.idle":"2021-06-13T07:56:39.137622Z","shell.execute_reply.started":"2021-06-13T07:56:36.450239Z","shell.execute_reply":"2021-06-13T07:56:39.13677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m spacy init config \"base_config.cfg\" --lang=\"en\" --pipeline=[\"transformer\",\"ner\"] --optimize=\"accuracy\" --gpu --force","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:56:39.139602Z","iopub.execute_input":"2021-06-13T07:56:39.139995Z","iopub.status.idle":"2021-06-13T07:56:47.187964Z","shell.execute_reply.started":"2021-06-13T07:56:39.139957Z","shell.execute_reply":"2021-06-13T07:56:47.18697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import configparser\n\n# Read config file\nedit = configparser.ConfigParser()\nedit.read(\"./base_config.cfg\")\n\narch = edit['components.transformer.model']\narch[\"name\"] = '\"'+local_path+'\"'\n\nedit.remove_option('components.ner.model', 'no')\n\nedit.remove_option('training.optimizer', 'l2')\nedit.remove_option('training.optimizer', 'l2_is_weight_decay')\n\n# Write changes back to file\nwith open('./base_config.cfg', 'w') as configfile:\n    edit.write(configfile)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:56:47.191596Z","iopub.execute_input":"2021-06-13T07:56:47.191892Z","iopub.status.idle":"2021-06-13T07:56:47.202073Z","shell.execute_reply.started":"2021-06-13T07:56:47.191861Z","shell.execute_reply":"2021-06-13T07:56:47.201074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m spacy init fill-config \"base_config.cfg\" \"config.cfg\"","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:56:47.205077Z","iopub.execute_input":"2021-06-13T07:56:47.205372Z","iopub.status.idle":"2021-06-13T07:56:54.648922Z","shell.execute_reply.started":"2021-06-13T07:56:47.205346Z","shell.execute_reply":"2021-06-13T07:56:54.647941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m spacy train \"config.cfg\" \\\n    --output \"./output\" \\\n    --paths.train \"train.spacy\" \\\n    --paths.dev \"validation.spacy\" \\\n    --gpu-id 0 \\\n    --nlp.batch_size 64 \\\n    --training.dropout 0.2 \\\n    --training.patience 0 \\\n    --training.max_steps 5000 \\\n    --training.logger.progress_bar True","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:06:01.892951Z","iopub.execute_input":"2021-06-13T07:06:01.893291Z","iopub.status.idle":"2021-06-13T07:47:22.186504Z","shell.execute_reply.started":"2021-06-13T07:06:01.893256Z","shell.execute_reply":"2021-06-13T07:47:22.185513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model\nspacy.require_gpu()\nnlp_best = spacy.load(R\"./output/model-best\")","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:47:41.589929Z","iopub.execute_input":"2021-06-13T07:47:41.590358Z","iopub.status.idle":"2021-06-13T07:47:42.645845Z","shell.execute_reply.started":"2021-06-13T07:47:41.590314Z","shell.execute_reply":"2021-06-13T07:47:42.644989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing the model\ndoc = nlp_best('The supply of PCR reagents, trained lab personnel and the availability of laboratories with sufficient biocontainment levels are major challenges of SARS-CoV-2 detection in developing countries, such as Indonesia (Younes et al., 2020) . Therefore, it is not surprising that the tested people per week is still lower than the World Health Organization (WHO) standard (World Health Organization, 2020b) . Recently, SARS-CoV-2 with the D614G mutation became the most frequently detected globally, including South East Asia region (Korber et al., 2020; Nguyen et al., 2020) . Interestingly, SARS-CoV-2 with the G614 variant had significantly higher infectious titers than the original D614 virus, and COVID-19 patients with the G614 variant had a higher viral load than patients without the mutation (Korber et al., 2020) . A recent study showed that the SARS-CoV-2 with the G614 variant revealed increased infectivity, competitive fitness, and transmission than the wild-type D614 virus in human airway epithelial cells and hamster (Hou et al., 2020) . However, this mutation was not associated with the severity of COVID-19 (Korber et al., 2020; Nguyen et al., 2020) . Here, we aimed:\\n(1) to report full-length genome sequences of SARS-CoV-2 collected from four COVID-19 patients in the Special Region of Yogyakarta and Central Java provinces, Indonesia;\\n(2) to compare the clade distribution of full-length genome sequences from Indonesia (n = 60) from March to September 2020; and (3) to perform phylogenetic analysis of SARS-CoV-2 complete genomes from different countries, including Indonesia.') # input sample text\nprint(\"Entities\", [(ent.text, ent.label_) for ent in doc.ents])","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:47:44.941602Z","iopub.execute_input":"2021-06-13T07:47:44.942062Z","iopub.status.idle":"2021-06-13T07:47:45.415904Z","shell.execute_reply.started":"2021-06-13T07:47:44.942018Z","shell.execute_reply":"2021-06-13T07:47:45.414936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spacy.displacy.render(doc, style=\"ent\", jupyter=True) # display in Jupyter","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:47:46.969164Z","iopub.execute_input":"2021-06-13T07:47:46.969495Z","iopub.status.idle":"2021-06-13T07:47:46.975891Z","shell.execute_reply.started":"2021-06-13T07:47:46.969461Z","shell.execute_reply":"2021-06-13T07:47:46.975007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}