{"cells":[{"metadata":{},"cell_type":"markdown","source":"🌈**in this notebook I am trying to visualize the Entire Publication Text Corpus and observe relation between datasets and Text**.🌈"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport pandas as pd\nfrom tqdm.autonotebook import tqdm\nimport os\nimport json\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def append_text(filename, train_files_path =train_files_path, output = 'text'):\n    json_path = os.path.join(train_files_path, (filename+ '.json'))\n    Heading = []\n    Content = []\n    Combined = []\n    with open (json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            Heading.append(data.get('section_title'))\n            Content.append(data.get('text'))\n            Combined.append(data.get('section_title'))\n            Combined.append(data.get('text'))\n            \n    all_headings = ' '.join(Heading)\n    all_contents = ' '.join(Content)\n    all_data = ' '.join(Combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(append_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords \nfrom nltk.tokenize import word_tokenize\nnltk.download('stopwords')\n%matplotlib inline\ndef cleantext(df): \n    \n    train_df['cleaned_text'] = train_df['text'].replace(r'\\'|\\\"|\\,|\\.|\\?|\\+|\\-|\\/|\\=|\\(|\\)|\\n|\"', '', regex=True)\n    train_df['cleaned_text'] = train_df['cleaned_text'].replace(\"  \", \" \")\n    \n    # convert tweets to lowercase\n    train_df['cleaned_text'] = train_df['cleaned_text'].str.lower()\n    \n     #remove_symbols\n    train_df['cleaned_text']  = train_df['cleaned_text'].replace(r'[^a-zA-Z0-9]', \" \", regex=True)\n    \n    #remove punctuations \n    train_df['cleaned_text'] = train_df['cleaned_text'].replace(r'[[]!\"#$%\\'()\\*+,-./:;<=>?^_`{|}]+',\"\", regex = True)\n    \n    #remove_URL(x):\n    train_df['cleaned_text']  = train_df['cleaned_text'].replace(r'https.*$', \"\", regex = True)\n    \n    #remove stopwords and words_to_remove\n    \n    mystopwords = set(stopwords.words('english'))\n    \n    train_df['fully_cleaned_text'] = train_df['cleaned_text'].apply(lambda x: ' '.join([word for word in str(x).split() if word not in mystopwords]))\n    \n\n    return df\n\ntrain_df = cleantext(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\nwords_in_text_by_dataset = defaultdict(list)\n\nfor _, row in train_df.iterrows():\n    words_in_text_by_dataset[row['dataset_title']].extend(row['fully_cleaned_text'].split())\n\n# Defining our word cloud drawing function\ndef wordcloud_draw(data, color = 'white'):\n    wordcloud = WordCloud(stopwords = STOPWORDS,\n                          background_color = color,\n                          width = 3000,\n                          height = 2000\n                         ).generate(' '.join(data))\n    plt.figure(1, figsize = (12, 8))\n    plt.imshow(wordcloud)\n    plt.axis('off')\n    plt.show()\n\nfor dataset_title in train_df['dataset_title'].unique():\n    print(\"WordCloud for Publications Text mentioning\", dataset_title, \":\")\n    wordcloud_draw(words_in_text_by_dataset[dataset_title])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}