{"cells":[{"metadata":{},"cell_type":"markdown","source":"![](https://i.ibb.co/gTQNCML/Screenshot-from-2021-03-27-17-26-18.png)"},{"metadata":{},"cell_type":"markdown","source":"# 1. Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nimport glob\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nfrom os.path import join\nfrom spacy import displacy\nfrom datetime import datetime\nfrom typing import Callable, Optional\nfrom fuzzywuzzy import fuzz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DIRECTORY TREE\nDATA_DIR = \"../input/coleridgeinitiative-show-us-the-data\"\n\n# spacy.displacy settings\nLABEL_DT = \"DT\"  # dataset_title\nCOLORS = {LABEL_DT: \"#FF0000\"}\nOPTIONS = {\"ents\": [LABEL_DT], \"colors\": COLORS}\n\ntrain_files = glob.glob(join(DATA_DIR, \"train/*.json\"))\ntest_files = glob.glob(join(DATA_DIR, \"test/*.json\"))\ntrain_file = join(DATA_DIR, \"train.csv\")\nid_to_path = {\n    os.path.split(path)[-1][:-5]: path for path in train_files\n}\nassert len(id_to_path) == len(train_files)\nassert os.path.isfile(train_file)\nprint(\"Train Atricles found\", len(train_files))\nprint(\"Test Articles found\", len(test_files))\n\ndef r_json(path):\n    with open(path) as fr:\n        doc = json.load(fr)\n    return doc\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(train_file)\ndf_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Dataset Label instances\nLet's gain some intuition about the difference between `dataset_title` and `dataset_label`"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.query(\"dataset_title != dataset_label\")[[\"dataset_title\", \"dataset_label\"]].drop_duplicates().head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_matches_for_instance(instance: pd.Series, clean_function: Optional[Callable] = None):\n    \"\"\"\n    1. Find corresponding article\n    2. Localize Dataset Name Instances (dataset_label)\n    3. Prepare data for spacy.displacy\n    \"\"\"\n    article = r_json(id_to_path[instance[\"Id\"]])\n    docs = []\n    pattern = instance[\"dataset_label\"]\n    if clean_function:\n        pattern = clean_function(pattern)\n    for part in article:\n        title = part[\"section_title\"]\n        txt = part[\"text\"]\n        if clean_function:\n            txt = clean_function(txt)\n        matches = list(re.finditer(pattern, txt))\n        if matches:\n            docs.append({\n                \"text\": txt,\n                \"ents\": [{\"start\": m.start(), \"end\": m.end(), \"label\": LABEL_DT} for m in matches],\n                \"title\": title,\n            })\n    return docs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#### CHANGE THIS ID TO BROWSE DATASET ####\nrow_idx = 0\n\n#### UNCOMMENT LINE BELOWE TO USE clean_text as cleaning policy\ndocs = find_matches_for_instance(df_train.iloc[row_idx])\n# docs = find_matches_for_instance(df_train.iloc[row_idx], clean_function=clean_text)\n\ndisplacy.render(docs, style=\"ent\", manual=True, options=OPTIONS, page=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Investigate cleaning policy\n\nUnfortunately `dataset_label` value does not match exactly witch the text in the article. We will investigate what are the main differences between them."},{"metadata":{"trusted":true},"cell_type":"code","source":"def summarize_cleaning_policy(clean_function=None):\n    start = datetime.now()\n    matched_ids = []\n    missing_ids = []\n    for idx in range(len(df_train)):\n        docs = find_matches_for_instance(df_train.iloc[idx], clean_function=clean_function)\n        if docs:\n            matched_ids.append(idx)\n        else:\n            missing_ids.append(idx)\n    print(f\"Processing time {datetime.now() - start}\")\n    print(f\"Matched ids: {len(matched_ids)}\")\n    print(f\"Missing ids: {len(missing_ids)}\")\n    \n    ### error analysis\n    if not missing_ids:\n        print(\"Every dataset instance matched! GREAT!\")\n    else:\n        for idx in range(min(len(missing_ids), 5)):\n            instance = df_train.iloc[missing_ids[idx]]\n            pattern = instance[\"dataset_label\"]\n            article = r_json(id_to_path[instance[\"Id\"]])\n            found = []\n            pat_len = len(pattern)\n            stride = 5\n            fuzz_threshold = 90  # range from 0-100, 100 means match\n            for part in article:\n                txt = part[\"text\"]\n                for start_idx in range(0, len(txt), stride):\n                    chunk = txt[start_idx:start_idx + pat_len + stride]\n                    # TODO: FIX fuzz.partial_ratio(\"aaa bbb\", \" \") = 100, which is not desired!\n                    if fuzz.partial_ratio(pattern, chunk) > fuzz_threshold:\n                        found.append(chunk)\n            print(f\"Dataset label = {pattern}\")\n            print(f\"Article Instances = {found}\")\n            print()\n    return matched_ids, missing_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"found_ids, missing_ids = summarize_cleaning_policy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"found_ids, missing_ids = summarize_cleaning_policy(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"found_ids, missing_ids = summarize_cleaning_policy(clean_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Conlusions\nAs we may notice there are at least 2 common cases why `dataset_label` label does not match its instance in article text.\n\n1. Some part of paper instance has additional characters like `(` parantheses.\n```\nPattern = SLOSH model\nInstances = ['s (SLOSH) model ']\n```\n2. Lowercase/uppercase convention is not consistent.\n```\nPattern = National Education Longitudinal Study\nInstances = [' using National Education Longitudinal stu', 'g National Education Longitudinal study of']\n```\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}