{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport os\nimport re\n\nimport pandas as pd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text: str) -> str:\n    return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"document_id = '0aa82fc3-d220-4fd5-940f-f82db0b9220a'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ = df.loc[df['Id'] == document_id]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_[['dataset_title', 'dataset_label']].values.flatten().tolist() # labels for this document","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sections = []\nwith open(f'../input/coleridgeinitiative-show-us-the-data/train/{document_id}.json') as f:\n    json_decode = json.load(f)\n    for data in json_decode:\n        sections.append(data['text'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sections)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"search_string = 'We used data from'\nsections_ids_with_search = []\nfor i, section in enumerate(sections):\n    if search_string in section:\n        sections_ids_with_search.append(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sections_ids_with_search","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sections[3].index(search_string)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sections[3][:394]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Why are not included as data sources for this document?","metadata":{}},{"cell_type":"markdown","source":"### Are they included as data sources for other documents? Let's see.","metadata":{}},{"cell_type":"code","source":"# ADNI ignored, because it is obviously there\n\ncandidates = [\n    'Adult Changes in Thought (ACT)',\n    'ACT',\n    'the Rush Memory and Aging Project'\n    'Rush Memory and Aging Project',\n    'MAP',\n    'Religious Orders Study',\n    'ROS',\n    'PITT',\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_ = [clean_text(c) for c in candidates] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['cleaned_label'].isin(['adni'])].shape # our label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['cleaned_label'].isin(candidates_)].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Is there a worry?","metadata":{}}]}