{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df[train_df.Id == 'de4c8260-cd87-431c-b425-70c9c44596d0']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nfnames = os.listdir('../input/coleridgeinitiative-show-us-the-data/train')\nlen(fnames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n#paragraph_num = []\nmydataframe = pd.DataFrame()\n\n\nfor file in tqdm(fnames):\n    #print('-'*52)\n    paper_path = os.path.join('../input/coleridgeinitiative-show-us-the-data/train', file)\n    #print(paper_path)\n    paper = pd.read_json(paper_path)\n    paper_id = file.replace('.json','')\n    find = train_df[train_df.Id == paper_id].dataset_label.unique()[0]\n    #print(train_df[train_df.Id == paper_id])\n    #print(find)\n    section_count = 0 \n    small = []\n    #print(paper)\n    for text in paper.iterrows(): #each section\n        section_count += 1\n        text_df = text[1]\n        \n        if find.lower() in text_df.text.lower():\n            for sent in text_df.text.split('.'):\n                if find.lower() in sent.lower():\n                    mydataframe = mydataframe.append({'Id':paper_id,'label_found':find.lower(),'section_title':text_df.section_title,'section_num':section_count,'total_section':len(paper),'sentence':sent,}, ignore_index=True)\n            small.append(text[0])\n    if small == []:\n        mydataframe = mydataframe.append({'Id':paper_id,'label_found':find.lower(),'section_title':'NONE','section_num':'NONE','total_section':len(paper),'sentence':'NONE',}, ignore_index=True)\n        #print(find.lower(), paper_id)\n    paragraph_num.append(small)\n    \n            \n        #print(text_df)\n#         flag = []\n#         for token in find.lower().split(' '):\n#             if token.lower() in text_df.text.lower():\n#                 flag.append(token)\n#             else:\n#                 pass\n#         if len(flag) > 1:\n#             print('\\n FOUND ', flag)\n \nmydataframe\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 0 \nfor i,v in enumerate(paragraph_num):\n    if v == []:\n        #print(i)\n        count += 1\ncount","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removes the label from the sentence \nc_sent = []\nfor row in mydataframe.iterrows():\n    temp = row[1].sentence.lower()\n    for word in row[1].label_found.split(' '):\n        temp = temp.replace(word,'')\n    c_sent.append(temp)\n    \n    \nmydataframe['clean_sentence'] = c_sent","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mydataframe[mydataframe.section_num == 'NONE'].groupby('label_found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# label_words = train_df.dataset_label.unique()\n# label_words = \" \".join(label_words)\n# label_words = label_words.split(' ')\n# label_words = set(label_words)\n\n# label_words = list(label_words)\n# label_words\n# #['Research','dataset', 'Study','Data','Collection']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import spacy \nimport matplotlib.pyplot as plt \nfrom wordcloud import WordCloud, STOPWORDS\ntext = []\n\n\nfor row in mydataframe.clean_sentence:\n    text.append(row)\n    \n\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2560, height=1440).generate(\" \".join(text))\n\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"par_len = []\nfor file in fnames:\n    paper_path = os.path.join('../input/coleridgeinitiative-show-us-the-data/train', file)\n    #print(paper_path)\n    paper = pd.read_json(paper_path)\n    par_len.append(len(paper))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(paragraph_num), len(par_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"par_df = pd.DataFrame(paragraph_num)\npar_df[0].value_counts().iloc[:10]\n# 9546 how many are in the first 5 sections\nprint('how many are in the first 5 sections ->',par_df[0].value_counts().head(5).sum())\n#how many are in the last 5 sections \n\npar_df[0].value_counts().head(5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for x in mydataframe.groupby('Id').section_num.sum():\n#    print(x)\n\nmydataframe.groupby('Id').section_num.value_counts()#.to_frame()\n\n#len(mydataframe.Id.unique())\n#0007f880-0a9b-492d-9a58-76eb0b0e0bd7  2.0            1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#print(mydataframe.groupby('Id').section_num.value_counts().iloc[:10])\n# 9546 how many are in the first 5 sections\nprint('how many are in the first 5 sections ->',mydataframe.groupby('Id').section_num.value_counts())\n#how many are in the last 5 sections \n\n#mydataframe.groupby('Id').section_num.value_counts().head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"par_df = pd.DataFrame({'sections':paragraph_num, 'length':par_len})\n\nnew_col = []\nfor row in par_df.iterrows():\n    small = []\n    for occur in row[1].sections:\n        small.append('%.3f'%(occur/row[1].length))\n    new_col.append(small)\n\n    \npar_df = pd.DataFrame({'sections':paragraph_num, 'length':par_len, 'where':new_col})\npar_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# do longer papers say it at the start or end???","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sum([1 for x in par_df.sections if len(x) == 1])\n#only have one rerence to the dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\n\nfind_list = list(train_df.dataset_label.unique())\nlarge = []\nfor file in tqdm(fnames):\n    #print('-'*52)\n    paper_path = os.path.join('../input/coleridgeinitiative-show-us-the-data/train', file)\n    #print(paper_path)\n    paper = pd.read_json(paper_path)\n    paper_id = file.replace('.json','')\n    small = []\n    for text in paper.iterrows():\n        text_df = text[1]\n        for find in find_list:\n            if find.lower() in text_df.text.lower():\n                small.append(find)\n    large.append(small)\n\nlarge","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(fnames), len(large)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sum([1 for l in large if len(l) ==0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}