{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this competition, we need to extract \"the mention of datasets\". <br>\nHowever, I don't have a clear answer as to what constitutes a mention at this stage.<br>\n\nTherefore, we will use known labels to dig deeper into the sentences that contain specific words but are not explicitly indicated in the train.csv.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport json\nfrom tqdm import tqdm\nimport re\ntqdm.pandas()\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport plotly.express as px","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN = \"../input/coleridgeinitiative-show-us-the-data/train\"\nTEST  = \"../input/coleridgeinitiative-show-us-the-data/test\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_sentense = []\nfor file in tqdm(os.listdir(TRAIN)):\n    \n    texts = []\n    \n    ids = file.split(\".\")[0]\n    file_path = os.path.join(TRAIN, file)\n    with open(file_path, \"r\") as f:\n        json_datasets = json.load(f)\n    \n    for json_dataset in json_datasets:\n        for k, v in json_dataset.items():\n            if k == \"text\":\n                texts.append(v)\n    \n    paper_sentense.append([ids, \" \".join(texts)])\n                ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_sentense_df = pd.DataFrame(paper_sentense, columns=[\"Id\", \"Sentense\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Contain AND Mentioned","metadata":{}},{"cell_type":"markdown","source":"Percentage of those that are judged to be actually mentioning the sentence among those that contain the sentence.","metadata":{}},{"cell_type":"markdown","source":"## Not Cleaned","metadata":{}},{"cell_type":"code","source":"labels = train_df[\"dataset_label\"].unique()\nmetioned_rate = []\nfor label in tqdm(labels):\n    mentioned_ids = train_df[train_df[\"dataset_label\"] == label][\"Id\"].unique().tolist()\n    paper_sentense_df[f\"is_{label}_contains\"] = paper_sentense_df[\"Sentense\"].str.contains(label)\n    mean = paper_sentense_df[paper_sentense_df[f\"is_{label}_contains\"]][\"Id\"].isin(mentioned_ids).mean()\n    metioned_rate.append([label, mean])\n    \nmetioned_rate_df = pd.DataFrame(metioned_rate, columns=[\"Label\", \"MentionedRate\"]).sort_values(\"MentionedRate\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(data_frame=metioned_rate_df,y=metioned_rate_df[\"Label\"],x=\"MentionedRate\", orientation = 'h', width=1200, height=3000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## cleaned","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"cleaned_pre_label\"] = train_df[\"cleaned_label\"].apply(lambda x: x[:-1] if x[-1] == \" \" else x)\ncleaned_labels = train_df[[\"cleaned_label\", \"cleaned_pre_label\"]].drop_duplicates()\ncleaned_metioned_rate = []\ncleaned_paper_sentense_df = paper_sentense_df[[\"Id\", \"Sentense\"]]\ncleaned_paper_sentense_df[\"Sentense\"] = cleaned_paper_sentense_df[\"Sentense\"].progress_apply(totally_clean_text) \nfor (label, pre_label) in tqdm(cleaned_labels.values):\n    mentioned_ids = train_df[train_df[\"cleaned_label\"] == label][\"Id\"].unique().tolist()\n    cleaned_paper_sentense_df[f\"is_{label}_contains\"] = cleaned_paper_sentense_df[\"Sentense\"].str.contains(pre_label)\n    mean = cleaned_paper_sentense_df[cleaned_paper_sentense_df[f\"is_{label}_contains\"]][\"Id\"].isin(mentioned_ids).mean()\n    cleaned_metioned_rate.append([label, mean])\n    \ncleaned_metioned_rate_df = pd.DataFrame(cleaned_metioned_rate, columns=[\"Label\", \"MentionedRate\"]).sort_values(\"MentionedRate\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(data_frame=cleaned_metioned_rate_df,y=cleaned_metioned_rate_df[\"Label\"],x=\"MentionedRate\", orientation = 'h', width=1200, height=3000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Regarding the known labels, most of them are included = mention, but some of them seem to be less so.<br>\n\nSome obvious dataset names(Ex: aging intergrated database agid) were found that were not \"mention\".","metadata":{}},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mentioned \"National Education Longitudinal Study\"","metadata":{}},{"cell_type":"code","source":"q1 = paper_sentense_df[paper_sentense_df[\"Sentense\"].str.contains(\"National Education Longitudinal Study\")]\na1 = train_df[train_df[\"dataset_title\"].str.contains(\"National Education Longitudinal Study\")][\"Id\"].unique()\nq1[\"is_answer\"] = q1[\"Id\"].isin(a1.tolist()) * 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## No Mentioned","metadata":{}},{"cell_type":"markdown","source":"13 IDs contain \"National Education Longitudinal Study\", but train.csv mentioned it.","metadata":{}},{"cell_type":"markdown","source":"b2ad2a12-78ce-4c65-ba18-e5b7a82cf449\n> Until recently, longitudinal analyses were limited to use of the **National Education Longitudinal Study** 1988 (NELS:88) data -a set that does not account for the \"second wave\" of accountability measures (Dee & Jacob, 2006)\n\nNot present in references","metadata":{}},{"cell_type":"markdown","source":"b6f4c581-74cd-4b58-8006-b891cffad287\t\n> Thus, this chapter provides a background for the two analyses on S&E program entrance and completion that are described in chapters 3 and 4. Chapter 2 describes the study approach used in the two analyses. The first analysis, in chapter 3, uses data from the **National Education Longitudinal Study** of 1988 (NELS:88)\n\nPresent in references","metadata":{}},{"cell_type":"markdown","source":"26f6d309-d11a-4335-ac48-27a33c9d6953\n> However, in contrast to these findings, cross-sectional studies of large national datasets tend to report a negative relationship between parental homework help and student achievement, including studies using data from the **National Education Longitudinal Study** (NELS:88) (e.g., Fan 2001;Fan and Chen 2001;McNeal 1999;Muller 1995;Peng and Wright 1994;Sui-Chu and Willms 1996) and other more recent analyses of national and state-level datasets (Moroni et al. 2015;Robinson and Harris 2014)\n\nNot present in references","metadata":{}},{"cell_type":"markdown","source":"## Mentioned","metadata":{}},{"cell_type":"markdown","source":"f8b03c87-9d1a-4f20-b76b-cb6c69d447b2\n> For this reason, these data are frequently provided to organization clients in repackaged form without attribution as to origin, and the extent of reliance on federal expertise and data collection capabilities is thus obscured. 6 Three longitudinal studies have been conducted to date: the National Longitudinal Study of the High School Graduating Class of 1972 (NLS-72), starting with a sample of 12th graders; the High School and Beyond Study (IISB), starting with two samples, one of the sophomore (grade 10) and a second of the senior (grade 12) class of 1980; and the **National Education Longitudinal Study** of 1988 (NELS-88), starting with a sample of students in grade 8 as of 1988\n\nNot present in references <br>\nNot in the text, but in the remarks","metadata":{}},{"cell_type":"markdown","source":"f8c8eb2a-4e31-4661-9fed-6ab4f3280ba4\n> Both surveys include a variety of items that have been used in other studies such as the General Social Survey and the **National Education Longitudinal Study** of 1988-92 that ask about religious participation and school experiences. Survey items measuring parenting style are based on similar items in the ECLS-Kindergarten Spring Parent Interview Questionnaire (http://nces.ed.gov/ecls/ kindergarten/questionnaires.asp) and the Child\\'s Report of Parent Behavior Inventory (Shaefer, 1965) . Questions about basic demographic characteristics of the respondents and family activities were also included in the surveys.\\nThe ESM, created by Csikszentmihalyi and colleagues (Csikszentmihalyi & Larson, 1987) , is a form of time diary that examines the daily lives of individuals, including what they are doing and how they feel\n\nNot present in references","metadata":{}},{"cell_type":"markdown","source":"167f0d62-9912-4279-be98-5fcac4836889\n> Using data from the **National Education Longitudinal Study** of 1988 (NELS) and the National Postsecondary Student Aid Study (NPSAS), this study examines access to postsecondary education by individuals in different income and test score groups. While many studies have found a statistically significant effect of income on college enrollment,\\' less attention has been paid to the effect of family income after controlling for student achievement. \n\nNot present in references(Maybe)","metadata":{}},{"cell_type":"markdown","source":"**Still not clear on the differences.**","metadata":{}},{"cell_type":"markdown","source":"# Include \"ADNI\" sentense and Include \"ADNI\" word","metadata":{}},{"cell_type":"code","source":"adnis = train_df[train_df[\"cleaned_label\"] == \"adni\"][\"Id\"].unique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_sentense_df[\"CleanedSentense\"] = paper_sentense_df[\"Sentense\"].progress_apply(totally_clean_text) \n\npaper_sentense_df[\"is_in_sentense_adni\"] = paper_sentense_df[\"CleanedSentense\"].str.contains(\"adni\") * 1\npaper_sentense_df[\"is_in_word_adni\"] = paper_sentense_df[\"CleanedSentense\"].apply(lambda x: \"adni\" in x.split()) * 1\npaper_sentense_df[\"is_adni\"] = paper_sentense_df[\"Id\"].isin(adnis) * 1\n\npaper_sentense_df[\"is_relate_adni\"] = paper_sentense_df[[\"is_in_sentense_adni\",\"is_in_word_adni\",\"is_adni\"]].sum(axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paper_sentense_df[\"is_relate_adni\"].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## existing as sentense, not existing as word, not label(1, 0, 0)","metadata":{}},{"cell_type":"markdown","source":"ID: 221312c6-31ad-406d-9da6-6e416a07f74b\n> m**adni**ck","metadata":{}},{"cell_type":"markdown","source":"## existing as sentense, not existing as word, label(1, 0, 1)","metadata":{}},{"cell_type":"markdown","source":"ID: 63031adf-e584-481b-8b78-957b17c513bf\n> Drapves5gWxySRtDrYbx5bv£Gr$**adnI**1!\n\n?????????????????????","metadata":{}},{"cell_type":"markdown","source":"## existing as sentense,  existing as word, not label(1, 1, 0)","metadata":{}},{"cell_type":"markdown","source":"ID: 91a1f72f-2884-48d9-a04d-9baae3e234b8\n> For example, an examination of **ADNI** participants over age 65 suggested that performance on Trails B would vary within age-specific subgroups in this cohort (see Fig. 7 showing mean and standard error for Trails B in four older age subgroups).","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}