{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Some dataset names in the data are written in the form:\n\n### dataset_name (abbreviation)\n\n    \nIn this notebook, I tried to find dataset names using regex.\n\nI made the submission only for results that are not in the training set.\n\nAlthough it does not have a very high score, it may help you get better results.\n\n---\n\n**How does it work?**\n\n1. Find possible results using regex: **\\(([A-Z]{2,}-?[A-Z]{1,}?[a-z]?)[\\);]**\n\n        (Gets abbreviations inside parantheses)\n        \n\n2. Find tokens before abbreviations\n\n3. Determine if they can be dataset names\n\n4. Remove some results if they contain specified strings\n\n5. Submit\n\n---\n\n**Observations**\n\n1. Some institution names are written in the same format, and they are difficult to differentiate. Some models might be submitting institution names.\n\n    Therefore, removing them from your submissions may improve your results.\n\n2. While creating a training data for models, removing punctuations also removes useful information.\n\n3. Since parantheses come after the dataset name, lstms will miss this information while predicting for the dataset name.\n\n    Using bidirectional models or transformers might be a better idea. I will also try reversing the text.\n","metadata":{}},{"cell_type":"code","source":"import json\nimport pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport re\nfrom tqdm import tqdm\nfrom fuzzywuzzy import fuzz\n\ntest_data_dir = r'/kaggle/input/coleridgeinitiative-show-us-the-data/test'\ntest_example_names = [fn.split('.')[0] for fn in os.listdir(test_data_dir)]\n\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nmetadata = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\nlabels = list(metadata.cleaned_label.unique())\nlabels = sorted(labels, key = len, reverse = True)\nlabels = [l.strip() for l in labels]\n\nprint(f'labels: {len(labels)}')\nsample_sub","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:17:21.769299Z","iopub.execute_input":"2021-05-26T12:17:21.770013Z","iopub.status.idle":"2021-05-26T12:17:22.0128Z","shell.execute_reply.started":"2021-05-26T12:17:21.769913Z","shell.execute_reply":"2021-05-26T12:17:22.01205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstop_words = stopwords.words('english')\n\nfor t in ['and', 'was', 'in']:\n    stop_words.remove(t)\n\nbanned_kw = [\n    'STEM', 'FDA', 'SSH', 'FSIZE', 'PET', 'NCATE', 'TESOL', 'AVHRR-OI',\n    'ICT',\n    'AAEA',\n    'BMI', 'ADGC', 'CDRSUM', 'NASS',\n    'MMSE', 'CDR', 'SPSS', 'LCRP', 'DML', 'ITU', 'DRI', 'CIPSEA', 'IEP', 'NCES', 'BCG', 'HLM', 'MLLW', 'FDG', 'MRMC', 'MEOW'\n]\n\nbanned_values = [\n    'laboratory', 'body mass index', 'admission test', 'neural networks', 'accuracy of', 'chain reaction', 'adversarial network',\n    'state exam', 'reform act', 'least', 'labeling', 'principal components analysis', 'independent components analysis', 'markov chain', 'monte carlo',\n    'bayesian information', 'family wise error', 'posterior anterior', 'Bidirectional Encoder', 'Morphometry', 'Integral', 'T2*weighted', 'T2-weighted',\n    'T2weighted', 'T1*weighted', 'T1-weighted', 'T1weighted', 'EMCI', 'Learning Test', 'Gradepoint average', 'doctor of', 'masters of',\n    'Expected Family Contribution', 'life in', 'Long Short Term', 'Long ShortTerm', 'LSTM', 'lipoprotein', 'Support Vector Machine', 'User Interface',\n    'National Institute of', 'glucose', 'Research Division', '%', 'Heating Weeks', 'Public Management', 'Theory', 'Middle East respiratory',\n    'Discriminant Analysis', 'boltzmann', 'Disease Control and Prevention', 'polymorphism', 'positron emission tomography', 'dorsolateral', 'Data Analysis System',\n    'Analysis Kit', 'Google', 'Principal Analysis', 'Cognitive Impairment', 'Analysis of Variance'\n    ]\n\nbanned_after_tokens = stop_words\n\nbanned_values = [b.lower() for b in banned_values]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:19:58.343026Z","iopub.execute_input":"2021-05-26T12:19:58.343445Z","iopub.status.idle":"2021-05-26T12:19:58.354514Z","shell.execute_reply.started":"2021-05-26T12:19:58.343412Z","shell.execute_reply":"2021-05-26T12:19:58.353565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\n_RE_COMBINE_WHITESPACE = re.compile(r\"\\s+\")\ndef make_single_whitespace(text):\n    return _RE_COMBINE_WHITESPACE.sub(\" \", text).strip()\n\ndef remove_punc(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt))\n\ndef load_test_example_by_name(name):\n    doc_path = os.path.join(test_data_dir, name + '.json')\n    with open(doc_path) as f:\n        data = json.load(f)\n    return data\n\ndef get_doc_id(doc_path):\n    return os.path.split(train_example_names[0])[-1].split('.')[0]\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:19:58.721791Z","iopub.execute_input":"2021-05-26T12:19:58.722462Z","iopub.status.idle":"2021-05-26T12:19:58.73272Z","shell.execute_reply.started":"2021-05-26T12:19:58.722401Z","shell.execute_reply":"2021-05-26T12:19:58.731552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_words_from_abbr_in_parantheses(match, doc_text):\n    try:\n        match_i = doc_text.index(f'({match})')\n    except:\n        try:\n            match_i = doc_text.index(f'({match};')\n        except:\n            match_i = doc_text.index(f'({match}')\n\n    match_nopunc = remove_punc(match).replace(' ', '')\n    n_tokens = len(match_nopunc.replace(' ', ''))\n\n    slice_start_i = max(match_i - 200, 0)\n    slice_end_i = min(match_i + len(match) + 40, len(doc_text))\n    doc_slice = doc_text[slice_start_i: slice_end_i]\n    \n    # Remove dates\n    doc_slice = re.sub(r'(19|20)[0-9][0-9]', ' ', doc_slice)\n\n    # Remove parantheses\n    doc_slice = doc_slice.replace('(', ' ').replace(')', ' ').replace(';', ' ').replace('-', '').replace('\\n', ' ')\n\n    tokens = doc_slice.split(' ')\n\n    tokens = [t for t in tokens if len(t) > 0]\n    match_token_i = tokens.index(match_nopunc)\n\n    if match_token_i - n_tokens <= 0:\n        return [], [], \"\"\n\n    start_i = match_token_i-n_tokens\n    end_i = match_token_i\n\n    # If a prev token started with uppercase, use it\n    try:\n        if start_i > 1:\n            if tokens[start_i-2][0].isupper():\n                start_i -= 2\n            else:\n                if tokens[start_i-1][0].isupper():\n                    start_i -= 1\n\n        word_tokens = tokens[start_i:end_i]\n\n        # Drop number token if it is coming first\n        if word_tokens[0].isdigit():\n            word_tokens = word_tokens[1:]\n\n        # Remove 2 lowercase tokens from start\n        for _ in range(2):\n            if word_tokens[0][0].islower():\n                word_tokens = word_tokens[1:]\n                start_i += 1\n\n    except IndexError:\n        print(f'IndexError for {match}')\n        return [], [], \"\"\n\n    after_token = \"\"\n    if len(tokens) > match_token_i + 1:\n        after_token = tokens[match_token_i + 1]\n\n    before_tokens = tokens[max(start_i - 3, 0) : start_i]\n\n    return before_tokens, word_tokens, after_token\n\n\ndef tokens_are_dataset_name(tokens):\n    if len(tokens) == 0:\n        return False\n    \n    long_tokens = [t for t in tokens if len(t) > 3]\n    lowercase_count = len([t for t in long_tokens if t[0].islower()])\n    uppercase_count = len([t for t in long_tokens if t[0].isupper()])\n\n    return lowercase_count < 4 and uppercase_count > 0\n\ndef after_token_ok(after_token):\n    if after_token == \"\":\n        return True\n\n    if after_token.lower() in banned_after_tokens:\n        return False\n\n    # Probably plural\n    if after_token not in ['was', 'has', 'is', 'this'] and after_token[-1].lower() == 's':\n        return False\n\n    # Probably a link\n    if 'http' in after_token:\n        return False\n\n    # A reference. Datasets don't get referenced like that\n    if '[' in after_token and ']' in after_token:\n        return False\n\n    # Probably a link\n    if '/' in after_token:\n        return False\n\n    if 'cell' in after_token:\n        return False\n\n    return True\n\ndef before_tokens_ok(before_tokens):\n    if len(before_tokens) == 0:\n        return True\n\n    if 'by' in before_tokens:\n        return False\n\n    if 'adjusted' in before_tokens:\n        return False\n\n    return True\n\n\n#get_words_from_abbr_in_parantheses('BDNF', doc_text)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:19:59.419264Z","iopub.execute_input":"2021-05-26T12:19:59.419833Z","iopub.status.idle":"2021-05-26T12:19:59.61828Z","shell.execute_reply.started":"2021-05-26T12:19:59.419797Z","shell.execute_reply":"2021-05-26T12:19:59.617036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_doc_results(doc_id):\n    doc_json = load_test_example_by_name(doc_id)\n    doc_text = ' '.join([s['text'] for s in doc_json])\n\n    re_find_par = r'\\(([A-Z]{2,}-?[A-Z]{1,}?[a-z]?)[\\);]'\n    matches = set(re.findall(re_find_par, doc_text))\n    \n    selected_mathces = {}\n    for m in matches:\n        try:\n            before_tokens, found_tokens, after_token = get_words_from_abbr_in_parantheses(m, doc_text)\n        except Exception as e:\n            print(f'Exception for {m}')\n            raise e\n\n        cond1 = tokens_are_dataset_name(found_tokens)\n        cond2 = after_token_ok(after_token)\n        cond3 = before_tokens_ok(before_tokens)\n\n        \"\"\"if not cond1:\n            print(f'{m} : Tokens {found_tokens} do not make a dataset name.')\n\n        if not cond2:\n            print(f'{m} : Aftertoken {after_token} was in banlist.')\n\n        if not cond3:\n            print(f'{m} : Beforetokens {before_tokens} were in banlist.')\"\"\"\n\n        if cond1 and cond2 and cond3:\n            selected_mathces[m] = ' '.join(found_tokens)\n\n    # Drop by keyword\n    matches_not_banned = {m: v for m, v in selected_mathces.items() if m not in banned_kw}\n\n    # Drop by text\n    matches_not_banned = {m: v for m, v in matches_not_banned.items() if not any([b for b in banned_values if b in v.lower()])}\n    \n    # Drop if last word is 'institute'\n    matches_not_banned = {m: v for m, v in matches_not_banned.items() if v.split(' ')[-1].lower() != 'institute'}\n\n    return matches_not_banned","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:19:59.61992Z","iopub.execute_input":"2021-05-26T12:19:59.620493Z","iopub.status.idle":"2021-05-26T12:19:59.633695Z","shell.execute_reply.started":"2021-05-26T12:19:59.620452Z","shell.execute_reply":"2021-05-26T12:19:59.632792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\nids = []\nfor index, row in sample_sub.iterrows():\n    test_id = row['Id']\n    try:\n        res = get_doc_results(test_id)\n        preds = [v for k,v in res.items()]\n        preds = [clean_text(p) for p in preds]\n        pred_string = '|'.join(preds)\n        test_preds.append(pred_string)\n            \n    except Exception as e:\n        test_preds.append(\"\")\n        print(e)\n        \n    ids.append(test_id)\n\nsub_df = pd.DataFrame(columns = ['Id', 'PredictionString'])\nsub_df['Id'] = ids\nsub_df['PredictionString'] = test_preds\nsub_df.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:20:00.369863Z","iopub.execute_input":"2021-05-26T12:20:00.370295Z","iopub.status.idle":"2021-05-26T12:20:00.403871Z","shell.execute_reply.started":"2021-05-26T12:20:00.370261Z","shell.execute_reply":"2021-05-26T12:20:00.40275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_colwidth = 1000\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:20:00.561635Z","iopub.execute_input":"2021-05-26T12:20:00.562058Z","iopub.status.idle":"2021-05-26T12:20:00.573446Z","shell.execute_reply.started":"2021-05-26T12:20:00.562021Z","shell.execute_reply":"2021-05-26T12:20:00.572243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"temp_1 = [x.lower() for x in metadata['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in metadata['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in metadata['cleaned_label'].unique()]\nexisting_labels = set(temp_1 + temp_2 + temp_3)\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:20:32.017187Z","iopub.execute_input":"2021-05-26T12:20:32.017602Z","iopub.status.idle":"2021-05-26T12:20:32.031548Z","shell.execute_reply.started":"2021-05-26T12:20:32.017568Z","shell.execute_reply":"2021-05-26T12:20:32.030303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"test_preds_only_unseen = []\nfor preds in test_preds:\n    tokens = preds.split('|')\n    kept_tokens = []\n    \n    for token in tokens:\n        has_similar = False\n        for l in existing_labels:\n            if jaccard(token, l) >= 0.5:\n                has_similar = True\n                break\n                \n        if not has_similar:\n            kept_tokens.append(token)\n            \n    test_preds_only_unseen.append('|'.join(kept_tokens))\n\nsub_df = pd.DataFrame(columns = ['Id', 'PredictionString'])\nsub_df['Id'] = ids\nsub_df['PredictionString'] = test_preds_only_unseen\nsub_df.to_csv('submission.csv', index = False)\n\nsub_df\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-05-26T12:20:33.107933Z","iopub.execute_input":"2021-05-26T12:20:33.108363Z","iopub.status.idle":"2021-05-26T12:20:33.154382Z","shell.execute_reply.started":"2021-05-26T12:20:33.108328Z","shell.execute_reply":"2021-05-26T12:20:33.153478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}