{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install packages","metadata":{"papermill":{"duration":0.027306,"end_time":"2021-06-13T15:03:22.87985","exception":false,"start_time":"2021-06-13T15:03:22.852544","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n#####################################################################################################################\n# Copy uploaded packages to working directory\n! cp -r ../input/sparknlp-3-1-0-installation/sparknlp_installation.xyz .\n! mv sparknlp_installation.xyz sparknlp_installation.tar.gz\n\n# Unzip the package & and remove *.tar\n! tar -xzvf sparknlp_installation.tar.gz > /dev/null\n! rm -rf ./sparknlp_installation.tar.gz\n# Copy spark 3.0.2\n! cp -r /kaggle/working/sparknlp_installation/spark-3.0.2-bin-hadoop2.7 /usr/local/\n! rm -rf /kaggle/working/sparknlp_installation/spark-3.0.2-bin-hadoop2.7\n######################################################################################################################\n# Copy Java to prepare Java_home, and sparknlp *.jar dependencies\n! cp -r sparknlp_installation/java-8-openjdk-amd64 /usr/lib/jvm\n! rm -rf sparknlp_installation/java-8-openjdk-amd64\n# If CPU used\n#! cp -r ./offline_kaggle_installation/sparknlp_cpu_jar_packages/*.jar ./offline_kaggle_installation/spark-3.0.2-bin-hadoop2.7/jars\n# If GPU used\n! cp -r ./sparknlp_installation/sparknlp_gpu_jar_packages/*.jar /usr/local/spark-3.0.2-bin-hadoop2.7/jars\n! rm -rf ./sparknlp_installation/sparknlp_cpu_jar_packages\n! rm -rf ./sparknlp_installation/sparknlp_gpu_jar_packages\n! rm -rf ./sparknlp_installation/cache_pretrained/bert_base_cased_en_2.6.0_2.4_1598340336670\n! rm -rf ./sparknlp_installation/cache_pretrained/ner_dl_bert_en_2.6.0_2.4_1599550979101\n! rm -rf ./sparknlp_installation/cache_pretrained/ner_dl_en_2.4.3_2.4_1584624950746\n! rm -rf ./sparknlp_installation/nltk_data/corpora\n! rm -rf ./sparknlp_installation/nltk_data/taggers\n! rm -rf ./sparknlp_installation/nltk_data/tokenizers/punkt.zip\n! mv ./sparknlp_installation/nltk_data/tokenizers/punkt/english.pickle ./sparknlp_installation/nltk_data/tokenizers/punkt/english\n! mv ./sparknlp_installation/nltk_data/tokenizers/punkt/PY3/english.pickle ./sparknlp_installation/nltk_data/tokenizers/punkt/PY3/english\n! rm -rf ./sparknlp_installation/nltk_data/tokenizers/punkt/*.pickle\n! rm -rf ./sparknlp_installation/nltk_data/tokenizers/punkt/PY3/*.pickle\n#! rm -rf ./offline_kaggle_installation/nltk_data/tokenizers/punkt/PY3\n! rm -rf ./sparknlp_installation/nltk_data/tokenizers/punkt/README\n! mv ./sparknlp_installation/nltk_data/tokenizers/punkt/english ./sparknlp_installation/nltk_data/tokenizers/punkt/english.pickle\n! mv ./sparknlp_installation/nltk_data/tokenizers/punkt/PY3/english ./sparknlp_installation/nltk_data/tokenizers/punkt/PY3/english.pickle\n#! cp ../input/modin-installation/modin-0.9.1-py3-none-manylinux1_x86_64.whl ./offline_kaggle_installation/sparknlp_pyspark_installation\n#####################################################################################################################\n# Install py4j, pyspark, sparknlp, findspark\n!pip install py4j --no-index --find-links=file:///kaggle/working/sparknlp_installation/sparknlp_pyspark_installation\n!pip install pyspark --no-index --find-links=file:///kaggle/working/sparknlp_installation/sparknlp_pyspark_installation\n!pip install spark_nlp --no-index --find-links=file:///kaggle/working/sparknlp_installation/sparknlp_pyspark_installation\n!pip install findspark --no-index --find-links=file:///kaggle/working/sparknlp_installation/sparknlp_pyspark_installation\n#!pip install modin --no-index --find-links=file:///kaggle/working/offline_kaggle_installation/sparknlp_pyspark_installation\n! rm -r ./sparknlp_installation/sparknlp_pyspark_installation\n# ## For model train version 2\n# ! cp -r ../input/light-glove-dataset-model/light_dataset_glove_model .\n## For model train version 3\n#! cp -r ../input/best-dataset-glove-model/dataset_glove_model .\n\n# ## Best version up to now\n# ! cp -r ../input/v2-better-glove-dataset-model/dataset_glove_model .\n# A light version\n#! cp -r ../input/v1-small-dataset-glove-model/small_dataset_glove_model .\n\n! cp -r ../input/kagge-glove-model-v12/kagge_glove_model_v12 .","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":112.473675,"end_time":"2021-06-13T15:05:15.379211","exception":false,"start_time":"2021-06-13T15:03:22.905536","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:37:35.216027Z","iopub.execute_input":"2021-06-15T10:37:35.2165Z","iopub.status.idle":"2021-06-15T10:39:42.890921Z","shell.execute_reply.started":"2021-06-15T10:37:35.21642Z","shell.execute_reply":"2021-06-15T10:39:42.889769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Start SparkSession","metadata":{"papermill":{"duration":0.029495,"end_time":"2021-06-13T15:05:15.438401","exception":false,"start_time":"2021-06-13T15:05:15.408906","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Setup environment and check java version\nimport os\n%env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64\n%env SPARK_HOME=/usr/local/spark-3.0.2-bin-hadoop2.7\nos.environ[\"PYSPARK_SUBMIT_ARGS\"]=\"--master local[*] pyspark-shell\"\nos.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\nos.environ[\"SPARK_HOME\"] = \"/usr/local/spark-3.0.2-bin-hadoop2.7\"\nos.environ[\"PATH\"] = os.environ[\"JAVA_HOME\"] + \"/bin:\" + os.environ[\"PATH\"]\nos.environ[\"PATH\"] = os.environ[\"SPARK_HOME\"] + \"/bin:\" + os.environ[\"PATH\"]\nos.environ[\"LD_LIBRARY_PATH\"]=\"/usr/local/cuda-11.0/lib64:\"+os.environ[\"LD_LIBRARY_PATH\"]\nos.environ[\"PATH\"]=\"/usr/local/cuda-11.0/bin:\"+os.environ[\"PATH\"]\nimport findspark\nfindspark.init()\nimport sparknlp\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql.functions import col, udf, regexp_replace, input_file_name, rand\nfrom pyspark.sql import functions as F\nfrom pyspark.sql.types import *\nfrom sparknlp.annotator import *\nfrom sparknlp.common import *\nfrom sparknlp.base import *\nfrom sparknlp.pretrained import PretrainedPipeline, PipelineModel\nfrom pyspark.ml import Pipeline\nfrom sparknlp.base import LightPipeline\n\nspark = SparkSession.builder \\\n    .appName(\"Spark NLP\")\\\n    .master(\"local[*]\")\\\n    .config(\"spark.driver.memory\",\"16G\")\\\n    .config(\"spark.driver.maxResultSize\", \"0\") \\\n    .config(\"spark.kryoserializer.buffer.max\", \"2000M\")\\\n    .config(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\\\n    .config(\"spark.sql.execution.arrow.pyspark.fallback.enabled\", \"true\")\\\n    .config(\"spark.cleaner.referenceTracking.cleanCheckpoints\", True)\\\n    .config(\"spark.streaming.stopGracefullyOnShutdown\", True)\\\n    .getOrCreate()\n\n! java -version\n\nprint(\"Spark NLP version: {}\".format(sparknlp.version()))\nprint(\"Apache Spark version: {}\".format(spark.version))","metadata":{"papermill":{"duration":13.07973,"end_time":"2021-06-13T15:05:28.547539","exception":false,"start_time":"2021-06-13T15:05:15.467809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:39:42.893367Z","iopub.execute_input":"2021-06-15T10:39:42.893842Z","iopub.status.idle":"2021-06-15T10:39:57.855542Z","shell.execute_reply.started":"2021-06-15T10:39:42.893792Z","shell.execute_reply":"2021-06-15T10:39:57.854072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, json, re, gc\nimport pandas as pd\nimport nltk\n# nltk.load('./offline_kaggle_installation/nltk_data/tokenizers/punkt/english.pickle')\n# from nltk.tokenize import sent_tokenize\nimport pyarrow.parquet as pq\nimport pyarrow as pa\nimport pandas as pd\nimport numpy as np\nfrom tqdm.autonotebook import tqdm\ntqdm.pandas()\nimport dask.dataframe as dd\nimport multiprocessing as mp","metadata":{"papermill":{"duration":3.221119,"end_time":"2021-06-13T15:05:31.799619","exception":false,"start_time":"2021-06-13T15:05:28.5785","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:39:57.861492Z","iopub.execute_input":"2021-06-15T10:39:57.86203Z","iopub.status.idle":"2021-06-15T10:40:02.127444Z","shell.execute_reply.started":"2021-06-15T10:39:57.861983Z","shell.execute_reply":"2021-06-15T10:40:02.126375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set pipeline","metadata":{"papermill":{"duration":0.030007,"end_time":"2021-06-13T15:05:31.860625","exception":false,"start_time":"2021-06-13T15:05:31.830618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"document = DocumentAssembler()\\\n        .setInputCol(\"text\")\\\n        .setOutputCol(\"document\")\n\n# sentence = SentenceDetector()\\\n#         .setInputCols(['document'])\\\n#         .setOutputCol('sentence')\n\ntoken = Tokenizer()\\\n        .setInputCols(['document'])\\\n        .setOutputCol('token')\n\nglove_embeddings = WordEmbeddingsModel\\\n        .load('./sparknlp_installation/cache_pretrained/glove_100d_en_2.4.0_2.4_1579690104032') \\\n        .setInputCols([\"document\", \"token\"])\\\n        .setOutputCol(\"embeddings\")\n  \nloaded_ner_model = NerDLModel.load(\"./kagge_glove_model_v12\")\\\n        .setInputCols([\"document\", \"token\", \"embeddings\"])\\\n        .setOutputCol(\"ner\")\n\nconverter = NerConverter()\\\n        .setInputCols([\"document\", \"token\", \"ner\"])\\\n        .setOutputCol(\"ner_chunk\")\n\n\nner_prediction_pipeline = Pipeline(stages = [\n        document,\n        token,\n        glove_embeddings,\n        loaded_ner_model,\n        converter\n])\n\n\nempty_df = spark.createDataFrame([['']]).toDF('text')\n\npipeline_model = ner_prediction_pipeline.fit(empty_df)\nlightPipeline = LightPipeline(pipeline_model)\n","metadata":{"papermill":{"duration":12.901923,"end_time":"2021-06-13T15:05:44.793555","exception":false,"start_time":"2021-06-13T15:05:31.891632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:40:02.129409Z","iopub.execute_input":"2021-06-15T10:40:02.129965Z","iopub.status.idle":"2021-06-15T10:40:19.47086Z","shell.execute_reply.started":"2021-06-15T10:40:02.129861Z","shell.execute_reply":"2021-06-15T10:40:19.469439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# UDF functions","metadata":{"papermill":{"duration":0.17174,"end_time":"2021-06-13T15:05:45.136072","exception":false,"start_time":"2021-06-13T15:05:44.964332","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def sentence_detector(text):\n#     return sent_tokenize(text)\n\npd.set_option('display.max_columns', 150)\n\ndef sentence_detector(t):\n    t = str(t)\n    if(len(t) < 5):\n        return ['']\n    else:\n        # split .\n        a = re.split('\\.', t)   # split .\n        a = [x+'.' for x in a if x]   # add . in the last character of sentence\n        # find the sentence that length of them < 13\n        ind = [0]\n        for x in range(1, len(a)):\n            if len(a[x]) < 13:\n                ind.append(1)\n            else:\n                ind.append(0)\n        # take the index of them\n        indices = [i for i, x in enumerate(ind) if x == 0]\n        # distributed the sentence of paragraph on indices\n        take = []\n        if len(indices) == 1:\n            take.append([0, len(ind)])\n        else:\n            for i in range(len(indices)):\n                if i == len(indices)-1:\n                    take.append([indices[i], len(ind)])\n                else:\n                    take.append([indices[i], indices[i+1]])\n        # take the array of sentence before split . \n        b = []\n        for x in take:\n            tem = ''.join(a[x[0]:x[1]])\n            b.append(tem)\n        a = b\n        # find the sentence that not start by space or new line and followed is upper character\n        ind = [0]\n        for x in range(1, len(a)):\n            if re.match(' [A-Z]|\\n[A-Z]', a[x][:2]):\n                ind.append(0)\n            else:\n                ind.append(1)\n        # take the index of them    \n        indices = [i for i, x in enumerate(ind) if x == 0]\n        # distributed the sentence of paragraph on indices\n        take = []\n        if len(indices) == 1:\n            take.append([0, len(ind)])\n        else:\n            for i in range(len(indices)):\n                if i == len(indices)-1:\n                    take.append([indices[i], len(ind)])\n                else:\n                    take.append([indices[i], indices[i+1]])\n        # take the array of sentence before split .\n        b = []\n        for x in take:\n            tem = ''.join(a[x[0]:x[1]])\n            b.append(tem)\n        return b\n\nudf_sentence_detector = udf(sentence_detector, ArrayType(StringType()))\n\ndef get_ner(text):\n    entity = []\n    for ann in lightPipeline.fullAnnotate(text)[0]['ner_chunk']:\n        entity.append([ann.result, ann.metadata['entity'], ann.metadata['confidence']])\n    return entity\n\ndef containsAny(string, array):\n    if len(string) == 0:\n        return False\n    else:\n        return (any(word in string for word in array))\n\ncontains_udf = udf(containsAny, BooleanType())\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\nudf_clean_text = udf(clean_text, StringType())\n\ndef jaccard(str1, str2):\n    a = set(str1.lower().split())\n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\ndef filterSentence(sent):\n    sent = re.sub('XXXY|YXXX|XXXX|XXYY|YYYY', '', sent)\n    regex = r\"[A-Z][A-Za-z]+\"\n    matches = re.finditer(regex, sent)\n    words = {}\n    for matchNum, match in enumerate(matches):\n        words[f'{match.group()}'] = [match.start(), match.end()]\n#     print(words)\n    k = list(words.keys())\n    v = list(words.values())     \n    for x in k:\n        if re.match('[A-Z]{3,}', x):\n            return True\n    count = 0\n    for x in range(1, len(v)):\n        if abs(v[x][0] - v[x-1][1]) < 8:\n            count = count + 1\n            if count >= 1:\n                return True\n    return False\n\nudf_filter_sentence = udf(filterSentence, BooleanType())","metadata":{"papermill":{"duration":0.410594,"end_time":"2021-06-13T15:05:45.72504","exception":false,"start_time":"2021-06-13T15:05:45.314446","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:40:19.476238Z","iopub.execute_input":"2021-06-15T10:40:19.476613Z","iopub.status.idle":"2021-06-15T10:40:19.537057Z","shell.execute_reply.started":"2021-06-15T10:40:19.476566Z","shell.execute_reply":"2021-06-15T10:40:19.529583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load json text","metadata":{"papermill":{"duration":0.049297,"end_time":"2021-06-13T15:05:45.824774","exception":false,"start_time":"2021-06-13T15:05:45.775477","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ndir_train_path = '../input/coleridgeinitiative-show-us-the-data/train/'\ndir_test_path = '../input/coleridgeinitiative-show-us-the-data/test/'\n\n\ndef get_paragraph(dirpath, arr_data, fileid):\n    with open('{}'.format(dirpath+fileid)) as f:\n        t = json.load(f)\n#     i = max_id_to_index + 1\n    for x in t:\n        fileid = re.sub(\".json\", \"\", fileid)\n        text = str(x['text'])\n        # remove text in section that is very difficult to say using what data\n        section_removal = [\"unknown\"]\n        #section_removal = [\"result\", \"results\", \"appendix\", \"conclusion\", \"limitations\", \n        #                   \"limitation\", \"supplementary\", \"summary\", \"reference\", \"references\"]\n        section = x['section_title'].lower().split()\n        if (not any(s in section_removal for s in section) & (len(text) > 35)):\n            emoji_pattern = re.compile(\"[\"\n                                   u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                                   u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                                   u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                                   u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                                   \"]+\", flags=re.UNICODE)\n            text = emoji_pattern.sub(r'', text)\n            arr_data.append([fileid, text])\n#             i+=1\n            \n# Start collecting test json publications and save it to a parquet file\nfileids = os.listdir(dir_train_path)[1:500]\n#fileids = [\"bde73ad6-ce46-489b-8ee0-b162698cc6df.json\"]\narr_data = []\ni = 0\nfor fileid in fileids:\n    get_paragraph(dir_train_path, arr_data, fileid)\n    #i = len(arr_data)\n    \ntable = pa.Table.from_pandas(pd.DataFrame(arr_data, columns=['Id','text']), preserve_index=False)\npq.write_table(table, 'publication_text_test.parquet')\n\ndel [[table, arr_data]]\ngc.collect()\n\n# The objective of golden words is to reduce computing dimension and higher the accuracy\n#golden_words = pd.read_csv(\"../input/golden-words/golden_words.csv\").golden_words.tolist()\n#golden_words = [\"data\", \"codes\", \"database\", \"sample\", \"dataset\", \"survey\", \"using\"]\ngolden_words = [\"data\", \"codes\", \"database\", \"sample\", \"sampling\", \"datasets\", \"survey\", \"dataset\", \"iri\", \\\n                \"overland\",\"samples\", \"registry\", \"baseline\", \"temperature\", \"genome\", \"census\", \"noaa\", \"world\", \\\n                \"baltimore\", \"hub\", \"initiative\", \"international\", \"national\", \"baccalaureate\", \"beginning\", \\\n                \"coastal\", \"practices\", \"cccsl\", \"tide\", \"adni\", \"slosh\", \"c-cap\", \"ccap\", \"research\", \"covid\", \\\n                \"education\", \"aging\", \"longtitudinal\", \"science\", \"disease\", \"school\", \"doctorates\"]\n\n# golden_words = [\"data\", \"codes\", \"database\", \"sample\", \"samples\", \"sampling\", \"datasets\", \"survey\", \"dataset\", \"iri\", \"hurricanes\",\\\n#                 \"samples\", \"registry\", \"baseline\", \"temperature\", \"genome\", \"census\", \"noaa\", \"world\", \"baltimore\", \"hub\",\\\n#                 \"initiative\", \"alzheimer\", \"international\", \"national\", \"baccalaureate\", \"beginning\", \"coastal\", \"practices\", \"cccsl\",\\\n#                 \"program\", \"study\", \"network\", \"catalog\", \"tide\", \"adni\", \"from\"]\n# golden_verbs = [\"were used\", \"using\", \"was used\", \"used\",\"derived from\", \"derived\", \"collected from\", \"collected\", \\\n#                \"conducted\", \"gathered\", \"gathered from\", \"surveyed\", \"retrieved\", \"retrieved from\"\\\n#               \"extracted from\", \"obtained\", \"obtained from\", \"downloaded\", \"downloaded from\"]\ngolden_verbs = [\"used\", \"using\", \"uses\", \"derives\", \"derived\", \"collecte\", \"collected\", \"collecting\",\\\n                \"conducts\", \"conducted\", \"conducts\", \"conducting\", \"gather\", \"gathered\", \"gathering\", \\\n                \"surveyed\", \"retrieved\", \"retrieve\", \"retrieving\", \"retrieves\",\"received\", \"receives\", \"receiving\",\n               \"extracted\", \"extracting\", \"extract\", \"obtained\", \"obtaining\", \"downloaded\", \"downloading\", \\\n                \"acquired\", \"acquires\", \"acquiring\", \"acquire\", \"based\"]\n                \ngolden_words = golden_words + golden_verbs\n#print(golden_words)\ndata_to_test = spark.read.parquet(\"publication_text_test.parquet\")\\\n.filter(col(\"text\").isNotNull())\\\n.withColumn(\"length_sent\", F.length(col(\"text\"))).filter(col(\"length_sent\") > 35)\\\n.withColumn(\"sent\", udf_sentence_detector(\"text\"))\\\n.select(\"Id\", F.explode(col(\"sent\")).alias(\"text\"))\\\n.withColumn(\"text1\", F.regexp_replace(col(\"text\"), \"[^0-9a-zA-Z]+\", \" \"))\\\n.withColumn(\"token\", F.split(F.lower(col(\"text1\")), ' '))\\\n.withColumn(\"has_golden_words\", contains_udf(col(\"token\"), F.array([F.lit(i) for i in golden_words])))\\\n.filter(col(\"has_golden_words\") == \"true\")\\\n.withColumn(\"issentence\", udf_filter_sentence(col(\"text\")))\\\n.filter(col(\"issentence\") == True)\\\n.select(\"Id\", \"text\")\\\n.drop_duplicates()\n\ndata_to_test.repartition(380).write.mode(\"overwrite\").format(\"parquet\").save(\"cleaned_test_data\")","metadata":{"papermill":{"duration":18.739716,"end_time":"2021-06-13T15:06:04.613808","exception":false,"start_time":"2021-06-13T15:05:45.874092","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:40:19.539382Z","iopub.execute_input":"2021-06-15T10:40:19.539958Z","iopub.status.idle":"2021-06-15T10:41:09.591943Z","shell.execute_reply.started":"2021-06-15T10:40:19.539912Z","shell.execute_reply":"2021-06-15T10:41:09.590518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Truncate data to 2 parts","metadata":{}},{"cell_type":"code","source":"%%time\ndata_to_test_part1 = spark.read.parquet(\"cleaned_test_data\")\\\n.filter(F.size(F.split(col(\"text\"), \" \")) < 250)\n\ndata_to_test_part1.repartition(380).write.mode(\"overwrite\").format(\"parquet\").save(\"cleaned_test_data1\")","metadata":{"execution":{"iopub.status.busy":"2021-06-15T10:47:33.51476Z","iopub.execute_input":"2021-06-15T10:47:33.515226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_to_test_part2 = spark.read.parquet(\"cleaned_test_data\")\\\n.filter(F.size(F.split(col(\"text\"), \" \")) >250)\\\n.withColumn(\"text\", F.regexp_replace(col(\"text\"), \"\\s\", \" \"))\\\n.withColumn(\"text\", F.regexp_replace(col(\"text\"), \"[A-Za-z]\\d+|\\d+[A-Za-z]|\\d+|[/\\\\&\\.()<>\\-,%\\+\\*=\\]\\]\\'\\\"]\", \"\"))\\\n.withColumn(\"text\", F.regexp_replace(col(\"text\"), r\"\\b\\w{1,2}\\b\", \" \"))\\\n.withColumn(\"text\", F.regexp_replace(col(\"text\"), \" {2,}\", \" \"))\n\ndata_to_test_part2.repartition(380).write.mode(\"overwrite\").format(\"parquet\").save(\"cleaned_test_data2\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get dataset name","metadata":{"papermill":{"duration":0.031517,"end_time":"2021-06-13T15:06:04.798019","exception":false,"start_time":"2021-06-13T15:06:04.766502","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ndf1 = spark.read.parquet(\"cleaned_test_data1\")\ndf2 = spark.read.parquet(\"cleaned_test_data2\")\n\npipeline_model.transform(df1).write.mode(\"overwrite\").parquet(\"ner_model_output1\")\npipeline_model.transform(df2).write.mode(\"overwrite\").parquet(\"ner_model_output2\")\n#pipeline_model.transform(df).show()\n\ndf1_output = spark.read.parquet(\"ner_model_output1\")\ndf2_output = spark.read.parquet(\"ner_model_output2\")\ndf_output = df1_output.union(df2_output)\ndf_output.write.mode(\"overwrite\").parquet(\"ner_model_output\")\n","metadata":{"papermill":{"duration":22.532156,"end_time":"2021-06-13T15:06:27.366783","exception":false,"start_time":"2021-06-13T15:06:04.834627","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess from first NER output","metadata":{"papermill":{"duration":0.030719,"end_time":"2021-06-13T15:06:27.437663","exception":false,"start_time":"2021-06-13T15:06:27.406944","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ndf1 = spark.read.parquet(\"ner_model_output\")\n#df1.printSchema()\ndf1.withColumn(\"tmp\", F.explode(col(\"ner_chunk\")))\\\n.select(\"Id\", \"text\",\"tmp.*\").select(\"Id\", \"text\", \"result\", \"metadata.entity\", \"metadata.confidence\")\\\n.withColumnRenamed(\"result\", \"ner_extracted\")\\\n.withColumnRenamed(\"entity\", \"ner_label\").write.mode(\"overwrite\").parquet(\"ner_output\")","metadata":{"papermill":{"duration":0.791543,"end_time":"2021-06-13T15:06:28.260177","exception":false,"start_time":"2021-06-13T15:06:27.468634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:03.569318Z","iopub.execute_input":"2021-06-15T10:47:03.56982Z","iopub.status.idle":"2021-06-15T10:47:03.687719Z","shell.execute_reply.started":"2021-06-15T10:47:03.569778Z","shell.execute_reply":"2021-06-15T10:47:03.68651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_input_rest = pd.read_parquet(\"ner_output\", engine='pyarrow')\n# df_input_rest_ = pd.read_parquet(\"cleaned_test_data\", engine='pyarrow')\n# df_input_rest = dd.from_pandas(df_input_rest_, npartitions=16*mp.cpu_count())\n# df_input_rest[\"entity\"] = df_input_rest.map_partitions(lambda df: df.apply(lambda x: get_ner(x.text), axis=1), meta =(\"entity\",str))\n# df_input_rest = df_input_rest.compute(num_workers=4)\n# df_input_rest = df_input_rest[[\"Id\", \"text\", \"entity\"]]\n# df_input_rest = df_input_rest.explode(\"entity\")\n# df_input_rest = df_input_rest[df_input_rest.entity.notnull()]\n\n# df_input_rest['ner_extracted'] = df_input_rest.entity.str[0]\n# df_input_rest['ner_label'] = df_input_rest.entity.str[1]\n# df_input_rest['confidence'] = df_input_rest.entity.str[2]\n#df_input_rest =df_input_rest[df_input_rest.ner_label == \"DATASET_TITLE\"]\ndf_input_rest = df_input_rest[[\"Id\", \"text\", \"ner_extracted\", \"ner_label\", \"confidence\"]]\n#df_input_rest =df_input_rest[df_input_rest.ner_label == \"DATASET_TITLE\"]\ndf_input_rest = df_input_rest.rename(columns={\"entity\":\"ner_extracted\"})\nraw_output_ = df_input_rest.drop_duplicates(keep=\"first\")\ndel [[df_input_rest]]\ngc.collect()","metadata":{"papermill":{"duration":0.271527,"end_time":"2021-06-13T15:06:28.585068","exception":false,"start_time":"2021-06-13T15:06:28.313541","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:03.689626Z","iopub.execute_input":"2021-06-15T10:47:03.690085Z","iopub.status.idle":"2021-06-15T10:47:03.871013Z","shell.execute_reply.started":"2021-06-15T10:47:03.690042Z","shell.execute_reply":"2021-06-15T10:47:03.86963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Output Correction","metadata":{"papermill":{"duration":0.032013,"end_time":"2021-06-13T15:06:28.649458","exception":false,"start_time":"2021-06-13T15:06:28.617445","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndataset_title = list(train['dataset_title'])\ndataset_label = list(train['dataset_label'])\ncleaned_label = list(train['cleaned_label'])\n#all_labels = list(set(list(map(clean_text, set(dataset_title))) + list(map(clean_text, set(dataset_label)))))\nall_labels = list(set(dataset_title+dataset_label+cleaned_label))\nall_labels.append(\"Alzheimer's Disease Neuroimaging Initiative ADNI\")\nall_labels.append(\"SLOSH\")\nall_labels.append(\"NOAA C-CAP\")\nall_labels.append(\"NOAA CCAP\")\nall_labels.append(\"NOAA OISST\")\nall_labels.append(\"OASIS\")\nall_labels.append(\"HERD\")\nall_labels.append(\"National Inpatient Sample\")\nall_labels.append(\"Alzheimer's Disease Neuroimaging Initiative\")\nall_labels.append(\"Program for International Student Assessment\")\nall_labels.append(\"Progress in International Reading Literacy Study\")\nall_labels.append(\"SARS‐CoV‐2 genome sequences\")\nprint(\"Length of all_labels: \", len(all_labels))\n# Check if words has at least 2 capital letters\ndef isAllCap(words):\n    if sum([int(w.isupper()) for w in words]) > 1:\n        return True\n    else:\n        return False\n#print(\"test isAllCap:\", isAllCap(\"LINH\"))\n# jaccard\ndef jaccard(str1, str2):\n    a = set(str1.lower().split())\n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\n# calculate max jaccard of string with train all dataset lables\ndef max_jaccard_train(text):\n    #max_score = 0\n    if (text == None):\n        return 0\n    else:\n        jac_score = []\n        for label in all_labels:\n            jac_score.append(float(jaccard(text, label)))\n    return float(max(jac_score))\n\n#print(max_jaccard_train(\"covid\"))\n# Filter non-potential dataset\n# For example: Analytic data -> so it cannot be a dataset\ndef drop_non_potential_dataset(text):\n    nbr_words = text.split()\n    if ((text==\")\") | (text==\"(\")):\n        return None\n    elif ((len(nbr_words) == 2) & (\"data\" in nbr_words)):\n        nbr_words.remove(\"data\")\n        if isAllCap(nbr_words[0]):\n            return text\n        else: return None\n    else: return text\n        \n#print(drop_non_potential_dataset(\"BHB data\"))\n\n# correction, for example: and Overland Surges from Hurricanes\n\n# correction, for example: and Overland Surges from Hurricanes\ndef label_correction(txt):\n    if txt == None:\n        return None\n    else:\n        text = txt.lower()\n    if \"overland surges from hurricanes\" in text:\n        return \"NOAA Sea, Lake, and Overland Surges from Hurricanes\"\n    if \"and overland surges\" in text:\n        return \"NOAA Sea, Lake, and Overland Surges from Hurricanes\"\n    elif \"beginning postsecondary\" in text:\n        return \"Beginning Postsecondary Student\"\n    elif \"genome sequence of s\" in text:\n        return \"SARS-CoV-2 genome Sequence\"\n    elif \"genome sequences of s\" in text:\n        return \"SARS-CoV-2 genome Sequences\"\n    elif text.lower() == \"noaa tide\":\n        return \"NOAA tidal Station\"\n    elif \"continuum codes\" in text:\n        return \"Rural-Urban Continuum Codes\"\n    elif \"neuroimaging initiative\" in text:\n        return \"Alzheimer's Disease Neuroimaging Initiative ADNI\"\n    elif \"adni\" in text:\n        return \"ADNI\"\n    elif ((\"adni\" in text) & (\"initiative\" not in text)):\n        return \"ADNI\"\n    elif ((len(text.split()) < 4) & (\"alzheimer\" in text)):\n        return \"ADNI\"\n    elif \"storm surge inundation\" in text:\n        return \"NOAA Storm Surge Inundation\"\n    elif \"sea surface temperature\" in text:\n        return \"Optimum Interpolation Interpolation Sea Surface Temperature\"\n    elif \"oisst\" in text:\n        return \"Optimum Interpolation Interpolation Sea Surface Temperature\"\n    elif \"postsecondary student\" in text:\n        return \"Beginning Postsecondary Students\"\n    elif \"timss\" in text:\n        return \"Trends in International Mathematics and Science Study\"\n    elif \"pirls\" in text:\n        return \"PIRLS\"\n    elif \"ccap\" in text:\n        return \"NOAA C-CAP\"\n    elif \"c-cap\" in text:\n        return 'NOAA C-CAP'\n    elif \"iri\" in text.split():\n        return \"IRI Consumer Network Panel\"\n    elif \"cnp\" in text.split():\n        return \"IRI Consumer Network Panel\"\n    elif \"pisa\" in text.split():\n        return \"Program for International Student Assessment\"\n    elif \"slosh\" in text:\n        return \"SLOSH Model\"\n    elif \"herd\" in text:\n        return \"Higher Education Research and Development Survey\"\n    elif \"national assessment of education\" in text:\n        return \"National Assessment of Education Progress\"\n    elif \"trends in international mathematics\" in text:\n        return \"Trends in International Mathematics and Science Study\"\n    elif text == \"storm surge modeling\":\n        return \"slosh Model\"\n    elif \"nels\" in text:\n        return \"National Education Longitudinal Study\"\n    elif text == \"blsa\":\n        return \"Baltimore Longitudinal Study of Aging\"\n    elif text == \"nels\":\n        return \"National Education Longitudinal Study\"\n    elif text == \"naep\":\n        return \"National Assessment of Education Progress\"\n    elif \"staffing survey\" in text:\n        return \"Schools and Staffing Survey\"\n    elif text == \"schools and\":\n        return \"Schools and Staffing Survey\"\n    elif \"survey of earned doctorates\" in text:\n        return \"Survey of Earned Doctorates\"\n    elif \"whole-genome sequence of\" in text:\n        return \"Genome Sequence of SARS-COV-2\"\n    elif \"appendix\" in text:\n        return None\n    elif \"use\" in text:\n        return None\n    elif \"uses\" in text:\n        return None\n    elif \"using\" in text:\n        return None\n        return None\n    elif \"]\" in text:\n        return None\n    elif \"\\n\" in text:\n        return None\n    elif \"esi shoreline\" in text:\n        return \"ESI Shoreline\"\n    elif text == \"tide\":\n        return None\n    elif \"tide gauge\" in text:\n        return \"NOAA Tide Gauge\"\n    elif text == \"noaa\":\n        return None\n    else:\n        return txt\n    \n    \ndef voting(text):\n    text_list = text.split(\"|\")\n    voting = text_list\n    if (len(text_list)) > 1:\n        for dt in text_list:\n            if \"data\" in dt.split():\n                new_df = dt.replace(\"data\", \"database\")\n                for dt1 in text_list:\n                    if new_df == dt1:\n                        voting.remove(dt)\n                new_df = dt.replace(\"data\", \"dataset\")\n                for dt1 in text_list:\n                    if new_df == dt1:\n                        voting.remove(dt)\n            if \"database\" in dt.split():\n                new_df = dt.replace(\" database\", \"\")\n                for dt1 in text_list:\n                    if new_df == dt1:\n                        voting.remove(dt)\n                new_df = dt.replace(\"database\", \"dataset\")\n                for dt1 in text_list:\n                    if new_df == dt1:\n                        voting.remove(dt)\n        text_list = voting\n        for dt in text_list:\n            if \"data\" in dt.split():\n                new_df = dt.replace(\" data\", \"\")\n                for dt1 in text_list:\n                    if new_df == dt1:\n                        voting.remove(dt)\n    return \"|\".join([v for v in voting if v])\n\n#voting(\"early childhood longitudinal study|ecls k database|ecls k data\")\n\ndef isAllLower(text):\n    word_list = text.split()\n    if sum([int(w[0].isupper()) for w in word_list]) == 0:\n        return True\n    elif \"moisture\" in word_list:\n        return False\n    elif \"maps\" in word_list:\n        return False\n    elif ((word_list[-1] == \"survey\") & (len(word_list) > 2)):\n        return False\n    elif word_list[-1][0].isupper() == False:\n        return True\n    elif ((len(word_list) == 1) & (len(word_list[0]) > 6)):\n        return True\n    elif ((len(word_list) == 3) & (sum([int(w[0].isupper()) for w in word_list[0:1]]) == 0)):\n        return True\n    else:\n        return False\n    \n#isAllLower(\"For School achievement\")","metadata":{"papermill":{"duration":0.180966,"end_time":"2021-06-13T15:06:28.862149","exception":false,"start_time":"2021-06-13T15:06:28.681183","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:03.873007Z","iopub.execute_input":"2021-06-15T10:47:03.873495Z","iopub.status.idle":"2021-06-15T10:47:04.19699Z","shell.execute_reply.started":"2021-06-15T10:47:03.873436Z","shell.execute_reply":"2021-06-15T10:47:04.195637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.set_option('display.max_columns', 150, 'display.max_colwidth', None)\n# raw_output_ = df_input_rest","metadata":{"papermill":{"duration":0.039228,"end_time":"2021-06-13T15:06:28.934539","exception":false,"start_time":"2021-06-13T15:06:28.895311","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.198713Z","iopub.execute_input":"2021-06-15T10:47:04.199172Z","iopub.status.idle":"2021-06-15T10:47:04.206795Z","shell.execute_reply.started":"2021-06-15T10:47:04.199112Z","shell.execute_reply":"2021-06-15T10:47:04.202861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#text = \"The data used to support the findings of this study can be obtained from the Alzheimer's Disease Neuroimaging Initiative (ADNI) database (http://adni.loni.usc.edu/).\"\n#text = \"The data for this report were drawn from the first followup of the 1990 Beginning Postsecondary Students (BPS) Longitudinal Study conducted in the spring of 1992.\"\n#text = \"In addition to county centroid elevation and total county population as an offset term, each model contained the following covariates: 1) 9-level USDA 2013 Rural-Urban Continuum Codes (USDA RUCC), where higher-numbered categories indicate increasingly rural environments; 2) average number of persons per household, computed as county population/number of households; 3) interaction of persons per household and USDA Rural-Urban Continuum Codes; 4) independent state random effects; and 5) spatially-correlated county random effects.\"\n#text = \"In such cases, attribution based on HURDAT2 (or IBTrACS) alone will possibly introduce some uncertainties along the western coast section where TCs from both sides may coincide, and the effect is likely a weakening of the fluxes by coinciding TCs from both sides.\"\n#text = \"The Beginning Postsecondary Student also discusses mitigation activities that are currently being used throughout the State and potential strategies that should be considered in the future.\"\n#text = \"In this analysis, we used the Progress in International reading Literacy Study (PIrLS) to analyze images\"\n\n#text = \"Storm surge modeling data was obtained from the SLOSH display program and used to map the extent and depth of inundation resulting from various storm scenarios ( Figure 6).\"\n#text = \"Coastal flooding is the hazard with the longest history of hazard mitigation planning throughout the study region, primarily through participation in the National Flood Insurance Program (NFIP), which is administered by the Federal Emergency Management Agency (FEMA).\"\n# text =\"Comparative Indicators of Education in the United States and Other G-8 Countries: 2009 draws on the most current information about education from four primary sources: the Indicators of National Education Systems (INES) at the Organization for Economic Cooperation and Development (OECD); the Program for International Student Assessment (PISA)\"\n# get_ner(text)","metadata":{"papermill":{"duration":0.038208,"end_time":"2021-06-13T15:06:29.004775","exception":false,"start_time":"2021-06-13T15:06:28.966567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.209481Z","iopub.execute_input":"2021-06-15T10:47:04.210336Z","iopub.status.idle":"2021-06-15T10:47:04.219004Z","shell.execute_reply.started":"2021-06-15T10:47:04.210292Z","shell.execute_reply":"2021-06-15T10:47:04.217343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#raw_output_ = r","metadata":{"papermill":{"duration":0.038855,"end_time":"2021-06-13T15:06:29.075627","exception":false,"start_time":"2021-06-13T15:06:29.036772","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.221162Z","iopub.execute_input":"2021-06-15T10:47:04.221679Z","iopub.status.idle":"2021-06-15T10:47:04.232583Z","shell.execute_reply.started":"2021-06-15T10:47:04.221635Z","shell.execute_reply":"2021-06-15T10:47:04.230495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correction Process","metadata":{"papermill":{"duration":0.032008,"end_time":"2021-06-13T15:06:29.139704","exception":false,"start_time":"2021-06-13T15:06:29.107696","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\npd.set_option('display.max_columns', 150, 'display.max_colwidth', None)\nraw_output_[\"confidence\"] = pd.to_numeric(raw_output_[\"confidence\"], downcast=\"float\")\nraw_output_[\"ner\"] = raw_output_.ner_extracted.apply(drop_non_potential_dataset)\nraw_output_[\"ner\"] = raw_output_.ner.apply(label_correction)\nraw_output_ = raw_output_[raw_output_.ner.notnull()]\nraw_output_.drop(\"ner_extracted\", axis=\"columns\", inplace=True)\n#raw_output[\"max_jaccard\"] = raw_output.ner.apply(max_jaccard_train)\nraw_output = dd.from_pandas(raw_output_, npartitions=4*mp.cpu_count()) \nraw_output[\"max_jaccard\"] = raw_output.map_partitions(lambda df: df.apply(lambda x: max_jaccard_train(x.ner), axis=1), meta=(\"max_jaccard\", float))\n#df_input_rest = dd.from_pandas(df_input_rest_, npartitions=1*mp.cpu_count())\n#df_input_rest[\"entity\"] = df_input_rest.map_partitions(lambda df: df.progress_apply(lambda x: get_ner(x.text), axis=1), meta =(\"entity\",str))\nraw_output = raw_output.compute(num_workers=4)\n# del [[raw_output_]]\n# gc.collect()","metadata":{"papermill":{"duration":0.159439,"end_time":"2021-06-13T15:06:29.331247","exception":false,"start_time":"2021-06-13T15:06:29.171808","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.234514Z","iopub.execute_input":"2021-06-15T10:47:04.235158Z","iopub.status.idle":"2021-06-15T10:47:04.284389Z","shell.execute_reply.started":"2021-06-15T10:47:04.235052Z","shell.execute_reply":"2021-06-15T10:47:04.282887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_output[\"isAllLower\"] = raw_output.ner.apply(isAllLower)\n#raw_output = raw_output[(raw_output.isAllLower == False) & (raw_output.max_jaccard < 0.5)]\n#raw_output.drop('isAllLower', axis=1, inplace=True)\n#raw_output[raw_output.Id == \"b5282251-0d12-4a33-b332-84ae7128d3e0\"]","metadata":{"papermill":{"duration":0.040034,"end_time":"2021-06-13T15:06:29.405729","exception":false,"start_time":"2021-06-13T15:06:29.365695","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.285991Z","iopub.execute_input":"2021-06-15T10:47:04.286487Z","iopub.status.idle":"2021-06-15T10:47:04.330189Z","shell.execute_reply.started":"2021-06-15T10:47:04.286432Z","shell.execute_reply":"2021-06-15T10:47:04.32719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# raw_output = df_input_rest\n# raw_output[\"confidence\"] = pd.to_numeric(raw_output[\"confidence\"], downcast=\"float\")\n# raw_output[\"ner\"] = raw_output.ner_extracted.apply(drop_non_potential_dataset)\n# raw_output[\"ner\"] = raw_output.ner.apply(label_correction)\n# raw_output = raw_output[raw_output.ner.notnull()]\n# raw_output.drop(\"ner_extracted\", axis=\"columns\", inplace=True)\n# raw_output[\"max_jaccard\"] = raw_output.progress_apply(lambda x: max_jaccard_train(x.ner), axis=1)","metadata":{"papermill":{"duration":0.038681,"end_time":"2021-06-13T15:06:29.476571","exception":false,"start_time":"2021-06-13T15:06:29.43789","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.331715Z","iopub.status.idle":"2021-06-15T10:47:04.332443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_ner(text, confidence):\n    words = text.lower().split()\n    tf_return = False\n    if ((len(words) > 2) & (\"survey\" in text.lower())):\n        tf_return = True\n    if ((len(words) > 2) & (\"dataset\" in text.lower())):\n        tf_return = True\n    if ((len(words) > 2) & (\"datasets\" in text.lower())):\n        tf_return = True\n    if ((len(words) > 2) & (\"database\" in text.lower())):\n        tf_return = True\n    if ((len(words) > 2) & (\"archive\" in text.lower())):\n        tf_return = True\n    if ((len(words) > 2) & (\"sample\" in text.lower())):\n        tf_return = True\n    if ((confidence > 0.82)) & (len(words) > 3):\n        tf_return = True\n    if (len(words) > 8):\n        tf_return = False\n    return tf_return\n        \n        \n        \n#raw_output1 = raw_output[raw_output.ner_label.isin([\"DATASET_TITLE\", \"DATASET_LABEL\"])]\n# raw_output1 = raw_output1[raw_output1.confidence > 0.6]\nraw_output1 = raw_output\nraw_output1 = raw_output1[raw_output1.max_jaccard < 0.5]\n#raw_output1 = raw_output1[raw_output1.confidence > 0.8]\nraw_output1 = raw_output1[raw_output1.isAllLower == False]\nraw_output1[\"filter_1\"] = raw_output1.apply(lambda x: filter_ner(x.ner, x.confidence), axis=1)\nraw_output1 = raw_output1[raw_output1.filter_1 == True]\nraw_output1 = raw_output1[list(raw_output.columns)]\n#raw_output.drop('isAllLower', axis=1, inplace=True)\n\n\nraw_output2 = raw_output[raw_output.max_jaccard >= 0.5]\n\noutput = raw_output1.append(raw_output2)\noutput = output[[\"Id\", \"ner\"]].drop_duplicates(keep=\"first\")","metadata":{"papermill":{"duration":0.062486,"end_time":"2021-06-13T15:06:29.571266","exception":false,"start_time":"2021-06-13T15:06:29.50878","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.334391Z","iopub.status.idle":"2021-06-15T10:47:04.337545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef elect_outcome(text):\n    words_in_text = text.split(\"|\")\n    outcome= []\n    if len(words_in_text) < 2:\n        return text\n    else:\n        for x in range(len(words_in_text)):\n            for y in range(x+1, len(words_in_text)):\n                if ((words_in_text[x] != words_in_text[y]) & (jaccard(words_in_text[x], words_in_text[y]) >= 0.5) & (jaccard(words_in_text[x], words_in_text[y]) != 1)):\n                    if len(words_in_text[x]) > len(words_in_text[y]):\n                        outcome.append(words_in_text[y])\n                        outcome = list(set(outcome))\n                    else:\n                        outcome.append(words_in_text[x])\n                        outcome = list(set(outcome))\n        outcome = [w for w in words_in_text if w not in outcome]\n        return \"|\".join([w for w in outcome if w])\n\n             \n\ntext = 'adni|alzheimer s disease neuroimaging initiative adni|adrc dataset|wisconsin adrc dataset|w adrc dataset'\n                        \nprint(elect_outcome(text))","metadata":{"papermill":{"duration":0.043836,"end_time":"2021-06-13T15:06:29.647315","exception":false,"start_time":"2021-06-13T15:06:29.603479","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.33925Z","iopub.status.idle":"2021-06-15T10:47:04.341551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = output.rename(columns={\"ner\":\"PredictionString\"})\noutput[\"PredictionString\"] = output.PredictionString.apply(clean_text)\noutput = output[output.PredictionString.notnull()]\noutput = output.drop_duplicates(keep=\"first\")\noutput = output.groupby('Id').agg(lambda c: '|'.join(c)).reset_index()\noutput[\"PredictionString\"] = output.PredictionString.apply(voting)\noutput[\"PredictionString\"] = output.PredictionString.apply(elect_outcome)","metadata":{"papermill":{"duration":0.051231,"end_time":"2021-06-13T15:06:29.732676","exception":false,"start_time":"2021-06-13T15:06:29.681445","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.348217Z","iopub.status.idle":"2021-06-15T10:47:04.355599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#raw_output","metadata":{"papermill":{"duration":0.037706,"end_time":"2021-06-13T15:06:29.802989","exception":false,"start_time":"2021-06-13T15:06:29.765283","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.357451Z","iopub.status.idle":"2021-06-15T10:47:04.358534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Result","metadata":{"papermill":{"duration":0.03424,"end_time":"2021-06-13T15:06:29.869584","exception":false,"start_time":"2021-06-13T15:06:29.835344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dir_test_path = '../input/coleridgeinitiative-show-us-the-data/test'\nfiles = os.listdir(dir_test_path)\nlist_id = [re.sub(\".json\", \"\", f) for f in files]\nsubmit_std = pd.DataFrame(list_id, columns=[\"Id_std\"])\nresult = pd.merge(submit_std, output, left_on=\"Id_std\", right_on=\"Id\", how=\"left\")\nresult = result[[\"Id_std\", \"PredictionString\"]].rename(columns={\"Id_std\":\"Id\"})\nresult.to_csv('submission.csv', index=False)\n\npd.set_option('display.max_colwidth', None, 'display.max_rows', None)\noutput","metadata":{"papermill":{"duration":0.056069,"end_time":"2021-06-13T15:06:29.958029","exception":false,"start_time":"2021-06-13T15:06:29.90196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.360289Z","iopub.status.idle":"2021-06-15T10:47:04.361338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#raw_output","metadata":{"papermill":{"duration":0.038373,"end_time":"2021-06-13T15:06:30.029409","exception":false,"start_time":"2021-06-13T15:06:29.991036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T10:47:04.363287Z","iopub.status.idle":"2021-06-15T10:47:04.370533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output_id = list(set(output.Id))\n# output_input = list(set(spark.read.parquet(\"./cleaned_test_data\").select(\"Id\").distinct().toPandas()[\"Id\"]))\n# rawoutput_id = list(set(raw_output.Id))","metadata":{"papermill":{"duration":0.039258,"end_time":"2021-06-13T15:06:30.101831","exception":false,"start_time":"2021-06-13T15:06:30.062573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T04:40:39.282682Z","iopub.execute_input":"2021-06-15T04:40:39.283017Z","iopub.status.idle":"2021-06-15T04:40:40.387604Z","shell.execute_reply.started":"2021-06-15T04:40:39.282977Z","shell.execute_reply":"2021-06-15T04:40:40.386755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.032919,"end_time":"2021-06-13T15:06:30.167456","exception":false,"start_time":"2021-06-13T15:06:30.134537","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# [w for w in output_input if w not in rawoutput_id]","metadata":{"papermill":{"duration":0.039658,"end_time":"2021-06-13T15:06:30.239735","exception":false,"start_time":"2021-06-13T15:06:30.200077","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T04:44:10.865096Z","iopub.execute_input":"2021-06-15T04:44:10.865432Z","iopub.status.idle":"2021-06-15T04:44:10.869543Z","shell.execute_reply.started":"2021-06-15T04:44:10.865403Z","shell.execute_reply":"2021-06-15T04:44:10.868418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [w for w in output_input if w not in output_id]","metadata":{"papermill":{"duration":0.038443,"end_time":"2021-06-13T15:06:30.310914","exception":false,"start_time":"2021-06-13T15:06:30.272471","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T04:44:13.197134Z","iopub.execute_input":"2021-06-15T04:44:13.197477Z","iopub.status.idle":"2021-06-15T04:44:13.203234Z","shell.execute_reply.started":"2021-06-15T04:44:13.197448Z","shell.execute_reply":"2021-06-15T04:44:13.202337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# raw_output[raw_output.Id == \"e0614647-0606-45bc-9b6b-bcc2fce45b96\"]","metadata":{"papermill":{"duration":0.038083,"end_time":"2021-06-13T15:06:30.382066","exception":false,"start_time":"2021-06-13T15:06:30.343983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T04:44:16.759643Z","iopub.execute_input":"2021-06-15T04:44:16.760019Z","iopub.status.idle":"2021-06-15T04:44:16.764058Z","shell.execute_reply.started":"2021-06-15T04:44:16.759979Z","shell.execute_reply":"2021-06-15T04:44:16.762806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result.to_csv('submission.csv', index=False)\n# with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n#     print(result)\n    \n# del [[result2, output, result]]\n# gc.collect()","metadata":{"papermill":{"duration":0.048485,"end_time":"2021-06-13T15:06:30.463045","exception":false,"start_time":"2021-06-13T15:06:30.41456","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T23:08:52.405774Z","iopub.execute_input":"2021-06-14T23:08:52.406235Z","iopub.status.idle":"2021-06-14T23:08:52.411316Z","shell.execute_reply.started":"2021-06-14T23:08:52.406206Z","shell.execute_reply":"2021-06-14T23:08:52.409906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! rm -rf publication_text_test.parquet\n! rm -rf ./cleaned_test_data\n! rm -rf ./cleaned_test_data1\n! rm -rf ./cleaned_test_data2\n! rm -rf ./sparknlp_installation\n! rm -rf ./ner_model_output1\n! rm -rf ./ner_model_output2\n! rm -rf ./ner_model_output\n! rm -rf ./ner_output\n! rm -rf ./kagge_glove_model_v12\n! ls .\nprint(\"it's done!\")","metadata":{"papermill":{"duration":4.69993,"end_time":"2021-06-13T15:06:35.195953","exception":false,"start_time":"2021-06-13T15:06:30.496023","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T11:55:27.142413Z","iopub.execute_input":"2021-06-14T11:55:27.142893Z","iopub.status.idle":"2021-06-14T11:55:27.149143Z","shell.execute_reply.started":"2021-06-14T11:55:27.14286Z","shell.execute_reply":"2021-06-14T11:55:27.14761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.034903,"end_time":"2021-06-13T15:06:35.265755","exception":false,"start_time":"2021-06-13T15:06:35.230852","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}