{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### In this notebook, I sorted sections by their title score.\n\n- Each title is scored by how many times sections with that title contained the first occurence of a dataset name.\n\n- The idea is, when you sort sections like this, uppermost sections have a higher chance of having a dataset name in them.\n\n- existing_labels: code from https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/232964","metadata":{}},{"cell_type":"code","source":"import json\nimport pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport re\nfrom tqdm import tqdm\nimport nltk\nimport random\nfrom nltk.tokenize import word_tokenize,sent_tokenize\n\ntrain_example_paths = glob.glob('../input/coleridgeinitiative-show-us-the-data/train/*.json')\ntrain_example_names = [fn.split('.')[0] for fn in os.listdir('../input/coleridgeinitiative-show-us-the-data/train')]\n\nmetadata = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndocIdx = train_example_names.copy()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-01T15:20:56.778459Z","iopub.execute_input":"2021-06-01T15:20:56.778812Z","iopub.status.idle":"2021-06-01T15:20:56.931587Z","shell.execute_reply.started":"2021-06-01T15:20:56.778783Z","shell.execute_reply":"2021-06-01T15:20:56.930427Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_train_example_by_name(name):\n    doc_path = os.path.join('../input/coleridgeinitiative-show-us-the-data/train', name + '.json')\n    with open(doc_path) as f:\n        data = json.load(f)\n    return data\n\ndef text_cleaning(text):\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text)).strip() # remove unnecessary literals\n\n    text = re.sub(r'\\[[0-9]+]', ' specialreference ', text)\n\n    # Remove years\n    text = re.sub(r'(19|20)[0-9][0-9]', ' specialyear ', text)\n\n    # remove other digits\n    text = re.sub(r'\\d+', ' ', text)\n\n    # remove extra spaces\n    text = re.sub(\"\\s+\",\" \", text)\n\n    # Remove websites\n    text = ' '.join(['specialwebsite' if 'http' in t or 'www' in t else t for t in text.split(' ') ])\n\n    return text.lower()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:21:16.846651Z","iopub.execute_input":"2021-06-01T15:21:16.847028Z","iopub.status.idle":"2021-06-01T15:21:16.854720Z","shell.execute_reply.started":"2021-06-01T15:21:16.846998Z","shell.execute_reply":"2021-06-01T15:21:16.853774Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\n\ntemp_1 = [text_cleaning(x) for x in metadata['dataset_label']]\ntemp_2 = [text_cleaning(x) for x in metadata['dataset_title']]\ntemp_3 = [text_cleaning(x) for x in metadata['cleaned_label']]\n\nexisting_labels = temp_1 + temp_2 + temp_3\nexisting_labels = [l.lower() for l in existing_labels]\nexisting_labels = list(set(existing_labels))\n\n# Sort labels by length in descending order\nexisting_labels = sorted(existing_labels, key = len, reverse= True)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:21:18.609816Z","iopub.execute_input":"2021-06-01T15:21:18.610188Z","iopub.status.idle":"2021-06-01T15:21:19.522604Z","shell.execute_reply.started":"2021-06-01T15:21:18.610157Z","shell.execute_reply":"2021-06-01T15:21:19.521682Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract Information","metadata":{}},{"cell_type":"code","source":"pos_sentences = []\nneg_sentences = []\ndoc_label_section_idx = []\ndoc_label_sentence_idx = []\ndoc_label_list = []\nfirst_label_sec_name = []\nfirst_labels = []\nn_secs = []\n\ndef process_doc(doc_id):\n    doc_json = load_train_example_by_name(doc_id)\n    this_doc_label_section_idx = []\n    this_doc_label_sentence_idx = []\n    this_doc_label_list = []\n    i_doc_sent = -1\n    doc_first_label = True\n    n_secs.append(len(doc_json))\n\n    for i_sec, section in enumerate(doc_json):\n        \n        sentences = sent_tokenize(section['text'])\n\n        adni_count = 0\n        for sentence in sentences:\n            i_doc_sent += 1\n            clean_sentence = text_cleaning(sentence)\n\n            has_label = False\n            label_is_adni = False\n            for clean_label in existing_labels:\n                if clean_label in clean_sentence:\n                    if doc_first_label:\n                        first_label_sec_name.append(section['section_title'])\n                        first_labels.append(clean_label)\n                        doc_first_label = False\n\n                    has_label = True\n                    this_doc_label_section_idx.append(i_sec)\n                    this_doc_label_sentence_idx.append(i_doc_sent)\n                    this_doc_label_list.append(clean_label)\n                    clean_sentence = clean_sentence.replace(clean_label, '')\n\n    doc_label_section_idx.append(this_doc_label_section_idx)\n    doc_label_sentence_idx.append(this_doc_label_sentence_idx)\n    doc_label_list.append(this_doc_label_list)\n    if doc_first_label:\n        first_label_sec_name.append('NOT FOUND')\n        first_labels.append('NOT FOUND')","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:21:19.524306Z","iopub.execute_input":"2021-06-01T15:21:19.524614Z","iopub.status.idle":"2021-06-01T15:21:19.537882Z","shell.execute_reply.started":"2021-06-01T15:21:19.524585Z","shell.execute_reply":"2021-06-01T15:21:19.536630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for doc_id in tqdm(docIdx):\n    process_doc(doc_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:21:19.789688Z","iopub.execute_input":"2021-06-01T15:21:19.790080Z","iopub.status.idle":"2021-06-01T15:32:42.346407Z","shell.execute_reply.started":"2021-06-01T15:21:19.790045Z","shell.execute_reply":"2021-06-01T15:32:42.345415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Section Title Info","metadata":{}},{"cell_type":"code","source":"def process_sec_name(text):\n    text = re.sub('[^A-Za-z]+', ' ', str(text)).strip() # remove unnecessary literals\n\n    # remove extra spaces\n    text = re.sub(\"\\s+\",\" \", text)\n\n    text = ' '.join([t for t in text.split(' ') if len(t) > 1])\n\n    return text.lower()\n\nsection_order = pd.Series(first_label_sec_name).value_counts().to_frame().reset_index()\nsection_order.columns = ['sec_name', 'cnt']\n\nsection_order.sec_name = section_order.sec_name.apply(lambda x: process_sec_name(x))\nsection_order = section_order.groupby('sec_name')['cnt'].sum().to_frame().reset_index()\nsection_order.columns = ['sec_name', 'cnt']\nsection_order = section_order.loc[section_order.sec_name.str.len() > 0]\n\n# Consolidate rows that contain 'data'\nsec_cons = section_order.sec_name.str.contains('data') | section_order.sec_name.str.contains('sample')\ncount_sum_data = section_order.loc[sec_cons, 'cnt'].sum()\nsection_order = section_order.loc[~sec_cons].reset_index(drop = True)\nsection_order.loc[len(section_order)] = ['data', count_sum_data]\n\n# Consolidate rows that contain 'study'\nsec_cons = section_order.sec_name.str.contains('study')\ncount_sum_data = section_order.loc[sec_cons, 'cnt'].sum()\nsection_order = section_order.loc[~sec_cons].reset_index(drop = True)\nsection_order.loc[len(section_order)] = ['study', count_sum_data]\n\nsection_order = section_order.loc[section_order.cnt > 10]\nsection_order = section_order.sort_values(by = 'cnt', ascending= False).reset_index(drop = True)\n\nsection_order.to_csv('section_order.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:36:04.600110Z","iopub.execute_input":"2021-06-01T15:36:04.600464Z","iopub.status.idle":"2021-06-01T15:36:04.735351Z","shell.execute_reply.started":"2021-06-01T15:36:04.600434Z","shell.execute_reply":"2021-06-01T15:36:04.734082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"section_order","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:36:05.807691Z","iopub.execute_input":"2021-06-01T15:36:05.808075Z","iopub.status.idle":"2021-06-01T15:36:05.821732Z","shell.execute_reply.started":"2021-06-01T15:36:05.808042Z","shell.execute_reply":"2021-06-01T15:36:05.820653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sort Sections","metadata":{}},{"cell_type":"code","source":"def sort_doc_sections(doc_secs):\n    # doc_secs must be a list of dicts with field'section_title'\n    for sec in doc_secs:\n        section_title = process_sec_name(sec['section_title'])\n        if len(section_title) < 4:\n            sec['score'] = 0\n        else:\n            sec_scores = section_order.loc[section_order.sec_name.str.contains(section_title) |\\\n                                           section_order.sec_name.apply(lambda x: x in section_title), 'cnt']\n            # sum scores of all matches\n            result_score = sec_scores.sum() if len(sec_scores) > 0 else 0\n\n            sec['score'] = result_score\n\n    return sorted(doc_secs, key = lambda x: x['score'], reverse = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:36:12.777346Z","iopub.execute_input":"2021-06-01T15:36:12.777701Z","iopub.status.idle":"2021-06-01T15:36:12.785354Z","shell.execute_reply.started":"2021-06-01T15:36:12.777669Z","shell.execute_reply":"2021-06-01T15:36:12.784086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example","metadata":{}},{"cell_type":"code","source":"ex_i = 100\n\ndoc_json = load_train_example_by_name(train_example_names[ex_i])\ndoc_json = [{'section_title': s['section_title']} for s in doc_json]\n\nprint(f'Section: {first_label_sec_name[ex_i]}')\nprint(f'Dataset Name: {first_labels[ex_i]}')\nsort_doc_sections(doc_json)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:43:03.499675Z","iopub.execute_input":"2021-06-01T15:43:03.500037Z","iopub.status.idle":"2021-06-01T15:43:03.520132Z","shell.execute_reply.started":"2021-06-01T15:43:03.500007Z","shell.execute_reply":"2021-06-01T15:43:03.519362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}