{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text analysis","metadata":{}},{"cell_type":"markdown","source":"Hi every body","metadata":{}},{"cell_type":"markdown","source":"Special thanks to helper notebooks 🙏🏻:\n1. 📝Coleridge Initiative-EDA📚& Baseline Model🎯\n2. Twitter sentiment Extaction-Analysis,EDA and Model","metadata":{}},{"cell_type":"markdown","source":"Import library and modules","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n\nimport os\nimport re\nimport json\nimport glob\nfrom collections import defaultdict\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\n\n\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nfrom collections import Counter\n\nfrom PIL import Image\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n\n\nfrom tqdm import tqdm\nimport os\nimport nltk\nimport spacy\nimport random\nfrom spacy.util import compounding\nfrom spacy.util import minibatch\n\n \n        \ndef random_colours(number_of_colors):\n    '''\n    Simple function for random colours generation.\n    Input:\n        number_of_colors - integer value indicating the number of colours which are going to be generated.\n    Output:\n        Color in the following format: ['#E86DA4'] .\n    '''\n    colors = []\n    for i in range(number_of_colors):\n        colors.append(\"#\"+''.join([random.choice('0123456789ABCDEF') for j in range(6)]))\n    return colors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's load the datasets\n\n","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A quick look at the train data\n\n","metadata":{}},{"cell_type":"code","source":"train.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no null Values in the train set\n\n","metadata":{}},{"cell_type":"code","source":"[print(f\"{col}:{len(train[col].unique())}\") for col in train.columns]   #finding unique values in each column","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract text from json file and plus its column to train csv file","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \ntrain['text'] = train['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, now we have our text appended in our train dataframe.","metadata":{}},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Also, we have the text of for the sample_submission file","metadata":{}},{"cell_type":"code","source":"sample_sub.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA**","metadata":{}},{"cell_type":"code","source":"train.describe()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets look at the distribution of cleaned_data in the train set\n\n","metadata":{}},{"cell_type":"code","source":"temp = train.groupby('cleaned_label').count()['text'].reset_index().sort_values(by='text',ascending=False)\ntemp.style.background_gradient(cmap='Purples')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's draw a Funnel-Chart for visualization\n\n","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(go.Funnelarea(\n    text =temp.cleaned_label,\n    values = temp.text,\n    title = {\"position\": \"top center\", \"text\": \"Funnel-Chart of cleaned_label Distribution\"}\n    ))\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can move your mouse on each color to see details!","metadata":{}},{"cell_type":"markdown","source":"**Cleaning the data**","metadata":{}},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n#     text = re.sub(\"/'+/g\", ' ', text)\n    \n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\ntrain['text'] = train['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Most Common words\n**","metadata":{}},{"cell_type":"code","source":"train['temp_list'] = train['cleaned_label'].apply(lambda x:str(x).split())\ntop = Counter([item for sublist in train['temp_list'] for item in sublist])\ntemp = pd.DataFrame(top.most_common(20))\ntemp.columns = ['Common_words','count']\ntemp.style.background_gradient(cmap='Blues')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='black',\n        stopwords=stopwords,\n        max_words=200,\n        max_font_size=40, \n        scale=3,\n        random_state=1 # chosen at random by flipping a coin; it was heads\n).generate(str(data))\n\n    fig = plt.figure(1, figsize=(15, 15))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()\n\nshow_wordcloud(train['cleaned_label'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(temp, x=\"count\", y=\"Common_words\", title='Commmon Words in cleaned_label', orientation='h', \n             width=700, height=700,color='Common_words')\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And for text","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='black',\n        stopwords=stopwords,\n        max_words=200,\n        max_font_size=40, \n        scale=3,\n        random_state=1 # chosen at random by flipping a coin; it was heads\n).generate(str(data))\n\n    fig = plt.figure(1, figsize=(15, 15))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()\n\nshow_wordcloud(train['text'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(temp, x=\"count\", y=\"Common_words\", title='Commmon Words in text', orientation='h', \n             width=700, height=700,color='Common_words')\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}