{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this Notebook\nThis is a first run through the compeition to try and understand the datatset and realise the problem at hand.","metadata":{}},{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\n\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom wordcloud import WordCloud, STOPWORDS\n\n# NLP\nimport spacy\nnlp = spacy.load('en_core_web_lg') # , disable=['parser', 'ner'])\nnlp.max_length = 40000000\nnlp.add_pipe(nlp.create_pipe('sentencizer'))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Description\n\ntrain.csv -labels and metadata for the training set\ntrain/tezt directory - the full text of the training/test set's publications in JSON format, broken into sections with section titles\n* `id` - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets.\n* `pub_title` - title of the publication (a small number of publications have the same title).\n* `dataset_title` - the title of the dataset that is mentioned within the publication.\n* `dataset_label` - a portion of the text that indicates the dataset.\n* `cleaned_label` - the dataset_label, as passed through the clean_text function from the [Evaluation page](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation).\n\nsample_submission.csv - a sample submission file in the correct format.\n* `Id` - publication id.\n* `PredictionString` - To be filled with equivalent of `cleaned_label` of train data.","metadata":{}},{"cell_type":"code","source":"RANDOM_SEED = 42","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp = spacy.load(\"en_core_web_lg\")\n# Merge noun phrases and entities for easier analysis\nnlp.add_pipe(nlp.create_pipe('merge_entities'))\nnlp.add_pipe(nlp.create_pipe('merge_noun_chunks'))\n\ndef find_conjunct_noun_chunks(text):\n    doc = nlp(text)\n    chunks = list(doc.noun_chunks)\n    \n    conjunct_groups = set()\n    \n    for chunk in chunks:\n        #print(type(chunk.root), chunk.root.i)\n        #print(type(chunk.root.head), chunk.root.head.i)\n        #print(chunk.text, list(chunk.noun_chunks), chunk.start, chunk.end, chunk.root.text, chunk.root.dep_, chunk.root.head.text, chunk.conjuncts)\n        if len(chunk.conjuncts) > 0:\n            group = tuple(sorted([chunk.text] + [s.text for s in chunk.conjuncts]))\n            conjunct_groups |= {group}\n            \n    return conjunct_groups\n\nsample_text = \"A number of longitudinal epidemiologic studies, including the Baltimore Longitudinal Study of Aging, the New Mexico Aging Process Study, and the Massachusetts Male Aging Study, have demonstrated age-related increases in the likelihood of developing hypogonadism.\"\nfind_conjunct_noun_chunks(sample_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_text = \"The index comprises two categories, respectively cognitive skill (the latest test results from the Progress in International Reading Literacy Study, PIRLS; the Trends in International Mathematics and Science Study, TIMSS; the Programme for International Student Assessment, PISA; the initial output from the Programme for the International Assessment of Adult Competencies, PIAAC) and educational attainment (the latest literacy rate and graduation rates at the upper secondary and tertiary level).\"\nfind_conjunct_noun_chunks(sample_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_appos_groups(text):\n    doc = nlp(text)\n    chunks = list(doc.noun_chunks)\n    \n    appos_groups = set()\n    \n    for chunk in chunks:\n        if chunk.root.dep_ == \"appos\":\n            appos_group = tuple(sorted([chunk.text, chunk.root.head.text]))\n            # print(appos_group)\n            appos_groups |= {appos_group}\n            \n    return appos_groups\n\nsample_text = \"The index comprises two categories, respectively cognitive skill (the latest test results from the Progress in International Reading Literacy Study, PIRLS; the Trends in International Mathematics and Science Study, TIMSS; the Programme for International Student Assessment, PISA; the initial output from the Programme for the International Assessment of Adult Competencies, PIAAC) and educational attainment (the latest literacy rate and graduation rates at the upper secondary and tertiary level).\"\nfind_appos_groups(sample_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_text = \"A number of longitudinal epidemiologic studies, including the Baltimore Longitudinal Study of Aging, the New Mexico Aging Process Study, and the Massachusetts Male Aging Study, have demonstrated age-related increases in the likelihood of developing hypogonadism.\"\ndoc = nlp(sample_text)\nchunks = list(doc.noun_chunks)\nfrom pprint import pprint\nprint(sample_text)\npprint(chunks)\nconjunct_groups = set()\nappos_groups = set()\n\nfor chunk in chunks:\n    print(type(chunk.root), chunk.root.i)\n    print(type(chunk.root.head), chunk.root.head.i)\n    print(chunk.text, list(chunk.noun_chunks), chunk.start, chunk.end, chunk.root.text, chunk.root.dep_,\n            chunk.root.head.text, chunk.conjuncts)\n    \n    if chunk.root.dep_ == \"appos\":\n        appos_group = tuple(sorted([chunk.text, chunk.root.head.text]))\n        print(appos_group)\n        appos_groups |= {appos_group}\n\n    group = tuple(sorted([chunk.text] + [s.text for s in chunk.conjuncts]))\n    if len(group) > 1:\n        conjunct_groups |= {group}\n    \nprint(conjunct_groups)\nprint(appos_groups)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ents = [(e.text, e.start_char, e.end_char, e.label_) for e in doc.ents]\npprint(ents)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from spacy import displacy\n\ndoc = nlp(sample_text)\ndisplacy.render(doc, style=\"dep\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's have a look at the training data csv file...","metadata":{}},{"cell_type":"code","source":"train_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's get the text data from json files and append them to the table.","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text', keep_list=False):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []    \n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n\n    if not keep_list:\n        headings = ' '.join(headings)\n        contents = ' '.join(contents)\n        combined = '\\n\\n '.join(combined)\n    \n    if output == 'text':\n        return contents\n    elif output == 'head':\n        return headings\n    else:\n        return combined","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text_list'] = train_df['Id'].progress_apply(lambda fn: read_append_return(fn, keep_list=True))\ntrain_df['text'] = train_df['Id'].progress_apply(lambda fn: read_append_return(fn, keep_list=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_section_index_of_dataset(text_list, dataset_label):\n    for i, t in enumerate(text_list):\n        if dataset_label.lower() in t.lower():\n            return i\n        \n    return -10\n\n# find_section_index_of_dataset([\"abc\", \"def\", \"123\"], \"ef\")\n\nsection_indexes = train_df.progress_apply(lambda r: find_section_index_of_dataset(r['text_list'], r['dataset_label']), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np \nimport scipy\nprint(scipy.stats.describe(section_indexes))\nplt.hist(section_indexes, bins=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_sub.shape)\nsample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nglob.glob(test_files_path + \"/*\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = sample_sub.iloc[2]\nprint(r.text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's save the data now in case we needed that while creating model later.","metadata":{}},{"cell_type":"code","source":"train_df.to_csv('train_papers.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inspect examples","metadata":{}},{"cell_type":"code","source":"print(train_df.shape)\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.pub_title.nunique()) # 論文数 14271","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.dataset_title.nunique()) # 45しかない\ntrain_df.dataset_title.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = train_df.iloc[1000]\n\ndataset_label = r.dataset_label\ntext = r.text\n\nprint(r)\n\n\nimport re\n\nhtml_str = re.sub(f\"({re.escape(dataset_label)})\", r\"<b style='color:navy'>\\1</b>\", text, flags=re.IGNORECASE)\nhtml_str = re.sub(\"\\n\", r\"<br>\", html_str)\n\nfrom IPython.display import HTML\n\ndisplay(HTML(html_str))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training examples\n\nThis study used data from the **National Education Longitudinal Study** (NELS:88) to examine the effects of dual enrollment programs for high school students on college degree attainment. \n\nUsing the nationally representative, longitudinal **National Education Longitudinal Study** of 1988 (NELS-88) data set, a logistic regression model was used to examine the extent to which outcome variables were differentially associated with gender for students participating in special education. \n\n\nThis responsiveness allowed phase alignment with tidal predictions for a **NOAA tidal station** (New-\n\n(別セクションにある微妙に違うmention)\nAll other cages were removed after 3 d of exposure, during which they were submerged approxiniately 34% of the time based on NOAA (1996) tidal predictions. \n\n(モデル名?)\nWith P-Surge, thousands of **SLOSH model** runs are made, forced by hurricane model input parameters from normal distributions centered on the current NHC official forecast,\n\n\nData used in the preparation of this article were obtained from the **Alzheimers Disease Neuroimaging Initiative** (ADNI) database (adni.loni.usc.edu). \n\n(略称含むパターン)\nWe examined the relation between PMI and structural integrity of Purkinje cells in autopsy cases with accurate PMI documented from the **Baltimore Longitudinal Study of Aging (BLSA)**. \n\n(ケースがラベルと合わないパターン)\nHere, we choose locations corresponding to four **NOAA tide gauge** stations near each study site, Stations 8452660, 8531680, 8534720, and 8638863 for Narragansett Bay, Jamaica Bay, Atlantic City, and Norfolk, respectively (Table 2) .\n\nSchool characteristics used to compare CFST schools to other NC public schools at program inception come from the National Center for Education Statistics Common Core of Data Public School Universe (NCES-CCD) and the North Carolina Department of Public Instruction's School Report Card compiled by the North Carolina Education Research Data Center (NCERDC).","metadata":{}},{"cell_type":"code","source":"query = \"NOAA Tide Gauge\"\nrows = train_df[train_df.dataset_label == query]\n\nrows.head(50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text):\n    return text.lower()\ndef find_sents(text, query):\n    found_sents = []\n    query = clean_text(query)\n    text = clean_text(text)\n    for s in nlp(text).sents:\n        if query in s.text:\n            found_sents.append(s)\n            \n    return found_sents","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inspect test data","metadata":{}},{"cell_type":"code","source":"print(sample_sub.shape)\nsample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub[sample_sub.text.str.lower().str.contains(\"test\")]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = sample_sub.iloc[0]\n\ntext = r.text\n\nprint(r)\n\ndataset_label = \"covid\"\n\nimport re\n\nhtml_str = re.sub(f\"({re.escape(dataset_label)})\", r\"<b style='color:navy'>\\1</b>\", text, flags=re.IGNORECASE)\nhtml_str = re.sub(\"\\n\", r\"<br>\", html_str)\n\nfrom IPython.display import HTML\n\ndisplay(HTML(html_str))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tqdm.pandas()\n# sample_sub['text'] = sample_sub['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generating Word cloud","metadata":{}},{"cell_type":"code","source":"text = ' '.join(train_df['text'].sample(frac=0.3))\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2560, height=1440).generate(text)\n\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see there are metions of 'et al' pretty significantly in the test of papers. Which is in fact related to quoting papers. This this should be a significant factor in determining the citation titles. Let, hope so...","metadata":{}},{"cell_type":"markdown","source":"# Preparing text","metadata":{}},{"cell_type":"code","source":"def prepare_text(text, nlp=nlp):\n    '''\n    Returns the text after stop-word removal and lemmatization.\n    text - Sentence to be processed\n    nlp - Spacy NLP model\n    '''\n    doc = nlp(text)\n    lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n    lemmatized_sentence = ' '.join(lemma_list)\n    \n    return lemmatized_sentence","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tqdm.pandas()\n# train_df['text'] = train_df['text'].progress_apply(prepare_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\nThis is a very naive model based on the assumption that topics having names of label or dataset_title in their content most porobaby are citing the same sources.","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\n'''\nIdea below of also using the 'dataset_title' is burrowed from\nhttps://www.kaggle.com/josephassaker/coleridge-initiative-eda-na-ve-submission\n'''\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook is a work in progress... This is just a first pass through the data to see what is the situation, along with a very Naive model. Even with that, trust me this used to have LB = 1.0 at some point in time. 😆😛  \n\n**If you found this notebook useful and use parts of it in your work, please don't forget to show your appreciation by upvoting this kernel. That keeps me motivated and inspires me to write and share these public kernels.** 😊","metadata":{}}]}