{"cells":[{"metadata":{},"cell_type":"markdown","source":"# purpose\n\n- The dataset label must appear somewhere in the article. So, let's find out where does the answer located. Intuitativley, the label is likely to appeared in sections like \"dataset\", \"methodology\", .... Thus, we may ignore the irelavent sections.\n\n- In addition, I will give a statistics EDA of the sections. Are there a common section structure of these papers? \n\n"},{"metadata":{},"cell_type":"markdown","source":"# functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/jamesmcguigan/coleridge-string-literals\nimport numpy as np \nimport pandas as pd\nimport simplejson\nimport re\nimport pydash\nimport sys\nimport os\nfrom collections import defaultdict\nfrom typing import *\nfrom joblib import Parallel, delayed\nfrom glob import glob\n\n\ndef clean_text(text: str) -> str:               return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\ndef clean_texts(texts: List[str]) -> List[str]: return [ clean_text(text) for text in texts ] \n\ndef read_json(index: str, test_train=\"test\") -> Dict:\n    filename = f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/{index}.json\"\n    with open(filename) as f:\n        json = simplejson.load(f)\n    return json\n        \ndef json2text(index: str, test_train=\"test\") -> str:\n    json  = read_json(index, test_train)\n    texts = [\n        row[\"section_title\"] + \" \" + row[\"text\"] \n        for row in json\n    ]\n    texts = clean_texts(texts)\n    text  = \" \".join(texts)\n    return text\n\n\ndef extract_label(text: str, lookup: Dict[str, Set[str]]) -> str:\n    labels = []\n    for label, values in lookup.items():\n        if any([\n            value in text\n            for value in values\n        ]):\n            labels += [ label.strip() ]\n            \n    # label = \"|\".join(set(labels))  # multi label support\n    label = Counter(labels).most_common(1)[0][0] if len(labels) else \"\"  # single most-popular label\n    # print('extract_label', labels, '->', label)\n    return label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_words(text):\n    return len(text.split())\n\n\nimport plotly.graph_objects as go\nimport plotly.express as px","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# prepare dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nfrom tqdm.notebook import tqdm\ndatapath = Path('../input/coleridgeinitiative-show-us-the-data/') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## training set label and index"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(datapath / 'train.csv')\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## read json text"},{"metadata":{"trusted":true},"cell_type":"code","source":"sections_df = pd.DataFrame()\nif Path('./sections_df.pickle').exists():\n    sections_df = pd.read_pickle('./sections_df.pickle')\nelse:\n    # this will takes around 30 mins\n    for i, row in tqdm(train_df.iterrows()):\n        paper_df = pd.DataFrame(read_json(row['Id'], 'train'))\n        # add title to the front\n        sections = ['title'] + clean_texts(paper_df['section_title'].to_list())\n        texts = clean_texts([row['pub_title']] + paper_df['text'].to_list())\n        for sect, text in zip(sections, texts):\n            sections_df = sections_df.append({'id':row['Id'], 'section': sect, 'label_in_section': row['cleaned_label'] in text, 'word_count': count_words(text)}, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sections_df.to_pickle('./sections_df.pickle')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### statistics of sections info"},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of unique sections\nprint(f\"number of unique sections {sections_df['section'].nunique()} / {len(sections_df)}\")\n\nsection_counts = sections_df['section'].value_counts()\nprint('the top 20 sections')\nsection_counts.head(20)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Common sections like \"abstracts\", \"introduction\", \"results\", \"methods\", \"discussions\", \"conclusions\" are indeed popular"},{"metadata":{"trusted":true},"cell_type":"code","source":"section_counts.tail(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# probability of apparence of dataset label of each sections\nprob_labels = sections_df[['section','label_in_section']].groupby('section').mean()\n\nprob_labels = prob_labels.join(section_counts)\nprob_labels.columns=['prob label in section', 'section counts']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prob_labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- there are indeed many mis-formated section"},{"metadata":{"trusted":true},"cell_type":"code","source":"prob_labels.sort_values('section counts', ascending=False).head(20)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- dataset label seems to appear in many sections"},{"metadata":{"trusted":true},"cell_type":"code","source":"prob_labels.sort_values('prob label in section', ascending=False).head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- I should see the trend with constraints of section counts"},{"metadata":{"trusted":true},"cell_type":"code","source":"prob_labels[prob_labels['section counts'] > 20].sort_values('prob label in section', ascending=False).head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}