{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-05T04:50:48.579722Z","iopub.execute_input":"2021-06-05T04:50:48.580133Z","iopub.status.idle":"2021-06-05T04:51:01.576698Z","shell.execute_reply.started":"2021-06-05T04:50:48.580051Z","shell.execute_reply":"2021-06-05T04:51:01.576016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Notebooks that I Adapted:\n\n* https://www.kaggle.com/dineshkumaranbalagan/descriptive-analysis\n\n* https://www.kaggle.com/mlconsult/score-57ish-with-additional-govt-datasets\n\n* https://www.kaggle.com/armandmorin/show-us-data\n","metadata":{}},{"cell_type":"markdown","source":"# Introduction\n\n## Goal\n\n* The end goal is to do string matching of known datasets names in order to detect mentions of datasets in scientific publications.\n* To build a strong NLP model that can infer from context whether or not a piece of text in a publication is refering to the usage of a dataset or not.","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\n\nimport spacy\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:51:01.577661Z","iopub.execute_input":"2021-06-05T04:51:01.577898Z","iopub.status.idle":"2021-06-05T04:51:12.742945Z","shell.execute_reply.started":"2021-06-05T04:51:01.577864Z","shell.execute_reply":"2021-06-05T04:51:12.741221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Description\n\n         - train.csv- CSV file contains metadata of the publications\n         - train-JSON file contains publications that are referenced in train.csv\n         - test-CSV file contains publications for testing purpose\n         - sample_submission.csv-CSV file conatins publications IDs column and prediction columns\n\n**id** - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets.\n\n**pub_title** -title of the publication (a small number of publications have the same title).\n\n**dataset_title** -the title of the dataset that is mentioned within the publication.\n\n**dataset_label** -a portion of the text that indicates the dataset.\n\n**cleaned_label** -the dataset_label, as passed through the clean_text function from the Evaluation page.\n\n**PredictionString** -To be filled with equivalent of cleaned_label of train data","metadata":{}},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"markdown","source":"### Train Set","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:51:12.745236Z","iopub.execute_input":"2021-06-05T04:51:12.745584Z","iopub.status.idle":"2021-06-05T04:51:12.754029Z","shell.execute_reply.started":"2021-06-05T04:51:12.745555Z","shell.execute_reply":"2021-06-05T04:51:12.752317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read train data\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:51:12.758697Z","iopub.execute_input":"2021-06-05T04:51:12.759147Z","iopub.status.idle":"2021-06-05T04:51:12.985912Z","shell.execute_reply.started":"2021-06-05T04:51:12.759113Z","shell.execute_reply":"2021-06-05T04:51:12.984885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:51:12.987213Z","iopub.execute_input":"2021-06-05T04:51:12.987597Z","iopub.status.idle":"2021-06-05T04:51:12.997979Z","shell.execute_reply.started":"2021-06-05T04:51:12.987566Z","shell.execute_reply":"2021-06-05T04:51:12.995975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#apply the function to train data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(read_json_pub)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:51:12.999757Z","iopub.execute_input":"2021-06-05T04:51:13.000086Z","iopub.status.idle":"2021-06-05T04:52:47.929100Z","shell.execute_reply.started":"2021-06-05T04:51:13.000055Z","shell.execute_reply":"2021-06-05T04:52:47.927743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#recheck\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:47.930502Z","iopub.execute_input":"2021-06-05T04:52:47.930795Z","iopub.status.idle":"2021-06-05T04:52:47.946309Z","shell.execute_reply.started":"2021-06-05T04:52:47.930765Z","shell.execute_reply":"2021-06-05T04:52:47.944994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read Submission Data","metadata":{}},{"cell_type":"code","source":"#read submission data\nsubmission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsubmission_df['text'] = submission_df['Id'].progress_apply(read_json_pub)\n\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:47.949313Z","iopub.execute_input":"2021-06-05T04:52:47.949659Z","iopub.status.idle":"2021-06-05T04:52:48.023585Z","shell.execute_reply.started":"2021-06-05T04:52:47.949630Z","shell.execute_reply":"2021-06-05T04:52:48.022232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsubmission_df.to_csv('submission_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.025786Z","iopub.execute_input":"2021-06-05T04:52:48.026134Z","iopub.status.idle":"2021-06-05T04:52:48.046110Z","shell.execute_reply.started":"2021-06-05T04:52:48.026105Z","shell.execute_reply":"2021-06-05T04:52:48.044725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read Samples","metadata":{}},{"cell_type":"code","source":"#let's read the first sample\n\nimport json\nwith open('../input/coleridgeinitiative-show-us-the-data/train/d0fa7568-7d8e-4db9-870f-f9c6f668c17b.json') as f:\n    sample = json.load(f)\n    \nsample[:2]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.047626Z","iopub.execute_input":"2021-06-05T04:52:48.048000Z","iopub.status.idle":"2021-06-05T04:52:48.056200Z","shell.execute_reply.started":"2021-06-05T04:52:48.047970Z","shell.execute_reply":"2021-06-05T04:52:48.055141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Within the first section, this publication mentions that they used data from the National Education Longitudinal Study. So the task of this competition is to find string of 'dataset_title' within the 'text' body and return 'cleaned_label'.","metadata":{}},{"cell_type":"code","source":"#get all 'section_title'\nfor s in sample:\n    print(s['section_title'])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.057507Z","iopub.execute_input":"2021-06-05T04:52:48.057779Z","iopub.status.idle":"2021-06-05T04:52:48.074159Z","shell.execute_reply.started":"2021-06-05T04:52:48.057749Z","shell.execute_reply":"2021-06-05T04:52:48.072925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SCRUB","metadata":{}},{"cell_type":"markdown","source":"### Basic Text Cleaning\n\nBefore we can create a bag of words or vectorize each document, we need to clean it up and split each document into an array of individual words. Computers are very particular about strings. If we tokenized our data in its current state, we would run into the following problems:\n\n* Counting things that aren't actually words. \n* Punctuation and capitalization would mess up our word counts. We need to remove punctuation and capitalization, so that all words will be counted correctly.","metadata":{}},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.075371Z","iopub.execute_input":"2021-06-05T04:52:48.075643Z","iopub.status.idle":"2021-06-05T04:52:48.114573Z","shell.execute_reply.started":"2021-06-05T04:52:48.075615Z","shell.execute_reply":"2021-06-05T04:52:48.113599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://towardsdatascience.com/text-analysis-feature-engineering-with-nlp-502d6ea9225d\n\ndef text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.115693Z","iopub.execute_input":"2021-06-05T04:52:48.115973Z","iopub.status.idle":"2021-06-05T04:52:48.126183Z","shell.execute_reply.started":"2021-06-05T04:52:48.115945Z","shell.execute_reply":"2021-06-05T04:52:48.125193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clean pub_title text\ntqdm.pandas()\ntrain_df['pub_title'] = train_df['pub_title'].progress_apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:48.127483Z","iopub.execute_input":"2021-06-05T04:52:48.127752Z","iopub.status.idle":"2021-06-05T04:52:52.322776Z","shell.execute_reply.started":"2021-06-05T04:52:48.127722Z","shell.execute_reply":"2021-06-05T04:52:52.321740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clean dataset_title text\ntqdm.pandas()\ntrain_df['dataset_title'] = train_df['dataset_title'].progress_apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:52.323839Z","iopub.execute_input":"2021-06-05T04:52:52.324064Z","iopub.status.idle":"2021-06-05T04:52:53.011185Z","shell.execute_reply.started":"2021-06-05T04:52:52.324042Z","shell.execute_reply":"2021-06-05T04:52:53.009724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clean train text\ntqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T04:52:53.013080Z","iopub.execute_input":"2021-06-05T04:52:53.013451Z","iopub.status.idle":"2021-06-05T05:10:04.244819Z","shell.execute_reply.started":"2021-06-05T04:52:53.013411Z","shell.execute_reply":"2021-06-05T05:10:04.243869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#review\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:04.246525Z","iopub.execute_input":"2021-06-05T05:10:04.246945Z","iopub.status.idle":"2021-06-05T05:10:04.262141Z","shell.execute_reply.started":"2021-06-05T05:10:04.246904Z","shell.execute_reply":"2021-06-05T05:10:04.261241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\ntrain_df.to_csv('train_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:04.263499Z","iopub.execute_input":"2021-06-05T05:10:04.264090Z","iopub.status.idle":"2021-06-05T05:10:34.738752Z","shell.execute_reply.started":"2021-06-05T05:10:04.264047Z","shell.execute_reply":"2021-06-05T05:10:34.738066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EXPLORE","metadata":{}},{"cell_type":"code","source":"#get info\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:34.739892Z","iopub.execute_input":"2021-06-05T05:10:34.740263Z","iopub.status.idle":"2021-06-05T05:10:34.794102Z","shell.execute_reply.started":"2021-06-05T05:10:34.740232Z","shell.execute_reply":"2021-06-05T05:10:34.792357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check null values\ntrain_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:34.797107Z","iopub.execute_input":"2021-06-05T05:10:34.797517Z","iopub.status.idle":"2021-06-05T05:10:34.829992Z","shell.execute_reply.started":"2021-06-05T05:10:34.797476Z","shell.execute_reply":"2021-06-05T05:10:34.828341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get summary\ntrain_df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:34.832502Z","iopub.execute_input":"2021-06-05T05:10:34.833563Z","iopub.status.idle":"2021-06-05T05:10:35.802183Z","shell.execute_reply.started":"2021-06-05T05:10:34.833490Z","shell.execute_reply":"2021-06-05T05:10:35.801299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The Train dataset has 19,661 counts but only 14,316 unique 'Id' in the dataset. This means some 'Id' are duplicates, meaning some 'Id' use multiple datasets.\n\nThe 'pub_title' has 19,661 counts but has only 14,271 unique titles. This means some 'pub_titles' are duplicates. There are less 'pub_title' counts than 'Id' counts, meaning some 'pub_title' has multiple 'Id'.\n\nThe 'dataset_title' has 19,661 counts but has only 45 unique titles. This means some 'dataset_title' are used many times by different publications.\n\nThe 'dataset_label' has 19,661 counts but has only 130 unique labels. This means some 'dataset_label' are duplicates. There are less 'dataset_title' counts than 'dataset_label', meaning some 'dataset_title' are labeled differently by different publications.","metadata":{}},{"cell_type":"code","source":"print('Number of duplicates in Id:', train_df['Id'].duplicated().sum())\nprint('Number of duplicates in pub_title:', train_df['pub_title'].duplicated().sum())\nprint('Number of duplicates in dataset_title:', train_df['dataset_title'].duplicated().sum())\nprint('Number of duplicates in dataset_label:', train_df['dataset_label'].duplicated().sum())\nprint('Number of duplicates in cleaned_label:', train_df['cleaned_label'].duplicated().sum())","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:35.803567Z","iopub.execute_input":"2021-06-05T05:10:35.804079Z","iopub.status.idle":"2021-06-05T05:10:35.825760Z","shell.execute_reply.started":"2021-06-05T05:10:35.804037Z","shell.execute_reply":"2021-06-05T05:10:35.824873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check out duplicates\nid_duplicates = train_df['Id'] == '170113f9-399c-489e-ab53-2faf5c64c5bc'\ntrain_df.loc[id_duplicates][:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:35.832797Z","iopub.execute_input":"2021-06-05T05:10:35.833887Z","iopub.status.idle":"2021-06-05T05:10:35.896144Z","shell.execute_reply.started":"2021-06-05T05:10:35.833841Z","shell.execute_reply":"2021-06-05T05:10:35.895472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The same 'Id' and 'pub_title' of sample 14798 and 14799 are labeled differently as 'survey of earned doctorates' and 'national center for science and engineering' although they are the same dataset_title.","metadata":{}},{"cell_type":"code","source":"#check out duplicates\npub_title_duplicates = train_df['pub_title'] == 'science and engineering indicator 2014'\ntrain_df.loc[pub_title_duplicates][:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:35.898160Z","iopub.execute_input":"2021-06-05T05:10:35.898593Z","iopub.status.idle":"2021-06-05T05:10:35.955221Z","shell.execute_reply.started":"2021-06-05T05:10:35.898561Z","shell.execute_reply":"2021-06-05T05:10:35.954577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"'survey of science and engineering research' dataset_title is labeled differently as 'survey of science and engineering research' and 'national center for science and engineering'","metadata":{}},{"cell_type":"code","source":"#check out duplicates\ndataset_title_duplicates = train_df['dataset_title'] == 'alzheimers disease neuroimaging initiative adni'\ntrain_df.loc[dataset_title_duplicates][:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:35.956932Z","iopub.execute_input":"2021-06-05T05:10:35.957207Z","iopub.status.idle":"2021-06-05T05:10:35.981298Z","shell.execute_reply.started":"2021-06-05T05:10:35.957179Z","shell.execute_reply":"2021-06-05T05:10:35.979402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we see with samples 1450 and 12456, there are same 'text' with same 'Id' and same 'pub_title' but labeled different.","metadata":{}},{"cell_type":"markdown","source":"## Look At Dataset Metrics","metadata":{}},{"cell_type":"code","source":"def get_num_words_per_sample(sample_texts):\n    \"\"\"Returns the median number of words per sample given corpus.\n\n    # Arguments\n        sample_texts: list, sample texts.\n\n    # Returns\n        int, median number of words per sample.\n    \"\"\"\n    num_words = [len(s.split()) for s in sample_texts]\n    return np.median(num_words)\n\nprint('dataset_title median word count:', get_num_words_per_sample(train_df['dataset_title']))\nprint('cleaned_label median word count:', get_num_words_per_sample(train_df['cleaned_label']))\nprint('text median word count:', get_num_words_per_sample(train_df['text']))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:35.984108Z","iopub.execute_input":"2021-06-05T05:10:35.984547Z","iopub.status.idle":"2021-06-05T05:10:45.569598Z","shell.execute_reply.started":"2021-06-05T05:10:35.984504Z","shell.execute_reply":"2021-06-05T05:10:45.568112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the number of samples/number of words per sample ratio\nlen(train_df['dataset_title']) / get_num_words_per_sample(train_df['dataset_title'])","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:45.570994Z","iopub.execute_input":"2021-06-05T05:10:45.571292Z","iopub.status.idle":"2021-06-05T05:10:45.595742Z","shell.execute_reply.started":"2021-06-05T05:10:45.571263Z","shell.execute_reply":"2021-06-05T05:10:45.594790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Look At Each Feature Individually","metadata":{}},{"cell_type":"markdown","source":"### 'Id'","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (30, 20)),\n\nsns.countplot(y = train_df['Id'], \n              order = train_df['Id'].value_counts(ascending = False)[:20].index, \n              palette = 'Spectral')\nplt.ylabel('Id',fontsize = 20)\nplt.title('Id')\nplt.show()\n\n#save\nplt.savefig('Id.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:45.597429Z","iopub.execute_input":"2021-06-05T05:10:45.597835Z","iopub.status.idle":"2021-06-05T05:10:46.243520Z","shell.execute_reply.started":"2021-06-05T05:10:45.597778Z","shell.execute_reply":"2021-06-05T05:10:46.242257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 'pub_title'","metadata":{}},{"cell_type":"code","source":"train_df['pub_title'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:46.244728Z","iopub.execute_input":"2021-06-05T05:10:46.245046Z","iopub.status.idle":"2021-06-05T05:10:46.259896Z","shell.execute_reply.started":"2021-06-05T05:10:46.245016Z","shell.execute_reply":"2021-06-05T05:10:46.258590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['pub_title'].value_counts().head(10).to_frame()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:46.261589Z","iopub.execute_input":"2021-06-05T05:10:46.262032Z","iopub.status.idle":"2021-06-05T05:10:46.291611Z","shell.execute_reply.started":"2021-06-05T05:10:46.261992Z","shell.execute_reply":"2021-06-05T05:10:46.290585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a frequency distribution to see which words are used the most\nwords = list( train_df['pub_title'].values)\nstopwords = stopwords_list\nsplit_words = []\n\nfor word in words:\n    lo_w = []\n    list_of_words = str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\n\nfor wordlist in split_words:\n    allwords += wordlist\n    \n#get 100 most common words\nmostcommon = FreqDist(allwords).most_common(100)\nmostcommon","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:46.292975Z","iopub.execute_input":"2021-06-05T05:10:46.293243Z","iopub.status.idle":"2021-06-05T05:10:47.709507Z","shell.execute_reply.started":"2021-06-05T05:10:46.293215Z","shell.execute_reply":"2021-06-05T05:10:47.707773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot frequency distributions\nwordcloud = WordCloud(width = 1600, height = 800, \n                      background_color = 'black', \n                      colormap = 'Spectral', \n                      stopwords = stopwords_list).generate(str(mostcommon))\n\nfig = plt.figure(figsize = (20, 10), facecolor = 'white')\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Top 100 Most Common Words in dataset_title', fontsize = 30)\nplt.tight_layout()\n\n#save\nplt.savefig('pub_title_wordcloud.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:47.711320Z","iopub.execute_input":"2021-06-05T05:10:47.711667Z","iopub.status.idle":"2021-06-05T05:10:50.354934Z","shell.execute_reply.started":"2021-06-05T05:10:47.711637Z","shell.execute_reply":"2021-06-05T05:10:50.351322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 'dataset_title'","metadata":{}},{"cell_type":"code","source":"train_df['dataset_title'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:50.356959Z","iopub.execute_input":"2021-06-05T05:10:50.357317Z","iopub.status.idle":"2021-06-05T05:10:50.369505Z","shell.execute_reply.started":"2021-06-05T05:10:50.357282Z","shell.execute_reply":"2021-06-05T05:10:50.368056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['dataset_title'].value_counts().head(20).to_frame()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:50.371007Z","iopub.execute_input":"2021-06-05T05:10:50.371288Z","iopub.status.idle":"2021-06-05T05:10:50.398391Z","shell.execute_reply.started":"2021-06-05T05:10:50.371258Z","shell.execute_reply":"2021-06-05T05:10:50.397215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a frequency distribution to see which words are used the most\nwords = list( train_df['dataset_title'].values)\nstopwords = stopwords_list\nsplit_words = []\n\nfor word in words:\n    lo_w = []\n    list_of_words = str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\n\nfor wordlist in split_words:\n    allwords += wordlist\n    \n#get 100 most common words\nmostcommon = FreqDist(allwords).most_common(100)\nmostcommon","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:50.399711Z","iopub.execute_input":"2021-06-05T05:10:50.400159Z","iopub.status.idle":"2021-06-05T05:10:51.689849Z","shell.execute_reply.started":"2021-06-05T05:10:50.400113Z","shell.execute_reply":"2021-06-05T05:10:51.688158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot frequency distributions\nwordcloud = WordCloud(width = 1600, height = 800, \n                      background_color = 'black', \n                      colormap = 'Spectral', \n                      stopwords = stopwords_list).generate(str(mostcommon))\n\nfig = plt.figure(figsize = (20, 10), facecolor = 'white')\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Top 100 Most Common Words in dataset_title', fontsize = 30)\nplt.tight_layout()\n\n#save\nplt.savefig('dataset_title_wordcloud.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:51.691611Z","iopub.execute_input":"2021-06-05T05:10:51.692119Z","iopub.status.idle":"2021-06-05T05:10:54.349937Z","shell.execute_reply.started":"2021-06-05T05:10:51.692077Z","shell.execute_reply":"2021-06-05T05:10:54.348992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (30, 30)),\n\nsns.countplot(y = train_df['dataset_title'], \n              order = train_df['dataset_title'].value_counts().index, \n              palette = 'Spectral')\nplt.ylabel('dataset_title',fontsize = 30)\nplt.xticks(fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('dataset_title.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:54.351090Z","iopub.execute_input":"2021-06-05T05:10:54.351314Z","iopub.status.idle":"2021-06-05T05:10:55.432301Z","shell.execute_reply.started":"2021-06-05T05:10:54.351287Z","shell.execute_reply":"2021-06-05T05:10:55.430754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BiGram","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\n#get bigrams \nvectorizer = CountVectorizer(ngram_range = (2, 2))\n\n#matrix of ngrams\nngrams = vectorizer.fit_transform(train_df['dataset_title']) \nfeatures = (vectorizer.get_feature_names())\nprint('\\n\\nFeatures : \\n', features)\n\n#count frequency of ngrams\nprint('\\n\\nX1 : \\n', ngrams.toarray())\n  \n#apply TFIDF\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(ngram_range = (2, 2))\nngrams = vectorizer.fit_transform(train_df['dataset_title'])\nscores = (ngrams.toarray())\nprint('\\n\\nScores : \\n', scores)\n  \n#get top ranking features\nsums = ngrams.sum(axis = 0)\ndata1 = []\nfor col, term in enumerate(features):\n    data1.append( (term, sums[0,col] ))\nranking = pd.DataFrame(data1, columns = ['term','rank'])\nwords = (ranking.sort_values('rank', ascending = False))\nprint ('\\n\\nWords head : \\n', words.head(20))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:55.433366Z","iopub.execute_input":"2021-06-05T05:10:55.433673Z","iopub.status.idle":"2021-06-05T05:10:56.014248Z","shell.execute_reply.started":"2021-06-05T05:10:55.433639Z","shell.execute_reply":"2021-06-05T05:10:56.012709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis = 0)\n\n#list of ngrams\nvocab = vectorizer.vocabulary_\ndf_bigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse = True)\n            ).rename(columns = {0: 'frequency', 1: 'bigram'})\n\nplt.figure(figsize = (20, 10))\nsns.lineplot(x = df_bigram['bigram'][:60], y = df_bigram['frequency'][:60])\nplt.xticks(rotation = 90, fontsize = 16)\nplt.xlabel('Bigram',fontsize = 20)\nplt.ylabel('Frequency',fontsize = 20)\nplt.title('Dataset Title Bigram',fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('dataset_title_bigram.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:56.015729Z","iopub.execute_input":"2021-06-05T05:10:56.016123Z","iopub.status.idle":"2021-06-05T05:10:57.072932Z","shell.execute_reply.started":"2021-06-05T05:10:56.016079Z","shell.execute_reply":"2021-06-05T05:10:57.071400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TriGram","metadata":{}},{"cell_type":"code","source":"#get trigrams \nvectorizer = CountVectorizer(ngram_range = (3, 3))\n\n#matrix of ngrams\nngrams = vectorizer.fit_transform(train_df['dataset_title']) \nfeatures = (vectorizer.get_feature_names())\nprint('\\n\\nFeatures : \\n', features)\n\n#count frequency of ngrams\nprint('\\n\\nX1 : \\n', ngrams.toarray())\n  \n#apply TFIDF\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(ngram_range = (3,3))\nngrams = vectorizer.fit_transform(train_df['dataset_title'])\nscores = (ngrams.toarray())\nprint('\\n\\nScores : \\n', scores)\n  \n#get top ranking features\nsums = ngrams.sum(axis = 0)\ndata1 = []\nfor col, term in enumerate(features):\n    data1.append( (term, sums[0,col] ))\nranking = pd.DataFrame(data1, columns = ['term','rank'])\nwords = (ranking.sort_values('rank', ascending = False))\nprint ('\\n\\nWords head : \\n', words.head(60))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:57.074479Z","iopub.execute_input":"2021-06-05T05:10:57.074780Z","iopub.status.idle":"2021-06-05T05:10:57.593421Z","shell.execute_reply.started":"2021-06-05T05:10:57.074747Z","shell.execute_reply":"2021-06-05T05:10:57.592026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis = 0)\n\n#list of ngrams\nvocab = vectorizer.vocabulary_\ndf_trigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse = True)\n            ).rename(columns = {0: 'frequency', 1:'trigram'})\n\nplt.figure(figsize = (20, 10))\nsns.lineplot(x = df_trigram['trigram'][:60], y = df_trigram['frequency'][:60])\nplt.xticks(rotation = 90, fontsize = 16)\nplt.xlabel('Trigram',fontsize = 20)\nplt.ylabel('Frequency',fontsize = 20)\nplt.title('Dataset Title Trigram',fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('dataset_title_trigram.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:57.594666Z","iopub.execute_input":"2021-06-05T05:10:57.594969Z","iopub.status.idle":"2021-06-05T05:10:58.784585Z","shell.execute_reply.started":"2021-06-05T05:10:57.594936Z","shell.execute_reply":"2021-06-05T05:10:58.783276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find one-worded, two-worded, three-worded, four-worded, five-worded dataset title\none_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 1]\ntwo_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 2]\nthree_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 3]\nfour_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 4]\nfive_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 5]\nsix_worded_dataset_title = train_df[train_df['dataset_title'].str.split().apply(len) == 6]\n\n#create a bar plot\nfig, ax = plt.subplots(figsize = (10, 6))\nax.bar([1, 2, 3, 4, 5, 6], [one_worded_dataset_title.size,\n                         two_worded_dataset_title.size,\n                         three_worded_dataset_title.size,\n                         four_worded_dataset_title.size,\n                         five_worded_dataset_title.size,\n                            six_worded_dataset_title.size])\n\n#label the x-axis instances\nax.set_xticks([1, 2, 3, 4, 5, 6])\nax.set_xticklabels([\"one\", \"two\", \"three\", 'four', 'five', 'six'])\n\n# set the title and the xy-axis labels\nplt.title(\"Number of Words in Dataset Title\")\nplt.xlabel(\"Number of Words\")\nplt.ylabel(\"Dataset Title\")\n\n# display the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:58.785995Z","iopub.execute_input":"2021-06-05T05:10:58.786277Z","iopub.status.idle":"2021-06-05T05:10:59.085872Z","shell.execute_reply.started":"2021-06-05T05:10:58.786248Z","shell.execute_reply":"2021-06-05T05:10:59.084138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 'cleaned_label'","metadata":{}},{"cell_type":"code","source":"train_df['cleaned_label'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:59.087192Z","iopub.execute_input":"2021-06-05T05:10:59.087480Z","iopub.status.idle":"2021-06-05T05:10:59.098970Z","shell.execute_reply.started":"2021-06-05T05:10:59.087450Z","shell.execute_reply":"2021-06-05T05:10:59.097713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label'].value_counts().head(20).to_frame()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:59.101694Z","iopub.execute_input":"2021-06-05T05:10:59.102143Z","iopub.status.idle":"2021-06-05T05:10:59.119353Z","shell.execute_reply.started":"2021-06-05T05:10:59.102116Z","shell.execute_reply":"2021-06-05T05:10:59.118155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Frequency Distributions","metadata":{}},{"cell_type":"code","source":"#create a frequency distribution to see which words are used the most\nwords = list( train_df['cleaned_label'].values)\nstopwords = stopwords_list\nsplit_words = []\nfor word in words:\n    lo_w = []\n    list_of_words = str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist\n    \n#get 100 most common words\nmostcommon = FreqDist(allwords).most_common(100)\nmostcommon","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:59.120780Z","iopub.execute_input":"2021-06-05T05:10:59.121249Z","iopub.status.idle":"2021-06-05T05:10:59.660991Z","shell.execute_reply.started":"2021-06-05T05:10:59.121218Z","shell.execute_reply":"2021-06-05T05:10:59.659459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot frequency distributions\nwordcloud = WordCloud(width = 1600, height = 800, \n                      background_color = 'black', \n                      colormap = 'Spectral', \n                      stopwords = stopwords_list).generate(str(mostcommon))\n\nfig = plt.figure(figsize = (20, 10), facecolor = 'white')\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Top 100 Most Common Words in cleaned_label', fontsize = 30)\nplt.tight_layout()\n\n#save\nplt.savefig('cleaned_label_wordcloud.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:10:59.662721Z","iopub.execute_input":"2021-06-05T05:10:59.663082Z","iopub.status.idle":"2021-06-05T05:11:02.345403Z","shell.execute_reply.started":"2021-06-05T05:10:59.663051Z","shell.execute_reply":"2021-06-05T05:11:02.343751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (30, 40)),\n\nsns.countplot(y = train_df['cleaned_label'], \n              order = train_df['cleaned_label'].value_counts().index, \n              palette = 'Spectral')\nplt.ylabel('Cleaned Label',fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('cleaned_label.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:02.347095Z","iopub.execute_input":"2021-06-05T05:11:02.347451Z","iopub.status.idle":"2021-06-05T05:11:06.057856Z","shell.execute_reply.started":"2021-06-05T05:11:02.347412Z","shell.execute_reply":"2021-06-05T05:11:06.056747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The classes are highly imbalanced.","metadata":{}},{"cell_type":"markdown","source":"## BiGram\n\nAn n-gram means a sequence of n-words.\n\nSome English words occur together more frequently. So, in a text document we may need to identify such pair of words which will help in sentiment analysis. \n\nBigram is 2 consecutive words in a sentence.","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\n#get bigrams \nvectorizer = CountVectorizer(ngram_range = (2, 2))\n\n#matrix of ngrams\nngrams = vectorizer.fit_transform(train_df['cleaned_label']) \nfeatures = (vectorizer.get_feature_names())\nprint('\\n\\nFeatures : \\n', features)\n\n#count frequency of ngrams\nprint('\\n\\nX1 : \\n', ngrams.toarray())\n  \n#apply TFIDF\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(ngram_range = (2, 2))\nngrams = vectorizer.fit_transform(train_df['cleaned_label'])\nscores = (ngrams.toarray())\nprint('\\n\\nScores : \\n', scores)\n  \n#get top ranking features\nsums = ngrams.sum(axis = 0)\ndata1 = []\nfor col, term in enumerate(features):\n    data1.append( (term, sums[0,col] ))\nranking = pd.DataFrame(data1, columns = ['term','rank'])\nwords = (ranking.sort_values('rank', ascending = False))\nprint ('\\n\\nWords head : \\n', words.head(20))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:06.059487Z","iopub.execute_input":"2021-06-05T05:11:06.059871Z","iopub.status.idle":"2021-06-05T05:11:06.567197Z","shell.execute_reply.started":"2021-06-05T05:11:06.059829Z","shell.execute_reply":"2021-06-05T05:11:06.565951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis = 0)\n\n#list of ngrams\nvocab = vectorizer.vocabulary_\ndf_bigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse = True)\n            ).rename(columns = {0: 'frequency', 1: 'bigram'})\n\nplt.figure(figsize = (20, 10))\nsns.lineplot(x = df_bigram['bigram'][:60], y = df_bigram['frequency'][:60])\nplt.xticks(rotation = 90, fontsize = 16)\nplt.xlabel('Bigram',fontsize = 20)\nplt.ylabel('Frequency',fontsize = 20)\nplt.title('Cleaned Label Bigram',fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('cleaned_label bigram.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:06.569820Z","iopub.execute_input":"2021-06-05T05:11:06.570114Z","iopub.status.idle":"2021-06-05T05:11:07.461339Z","shell.execute_reply.started":"2021-06-05T05:11:06.570088Z","shell.execute_reply":"2021-06-05T05:11:07.460379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TriGram\n\nTrigram is 3 consecutive words in a sentence. ","metadata":{}},{"cell_type":"code","source":"#get trigrams \nvectorizer = CountVectorizer(ngram_range = (3, 3))\n\n#matrix of ngrams\nngrams = vectorizer.fit_transform(train_df['cleaned_label']) \nfeatures = (vectorizer.get_feature_names())\nprint('\\n\\nFeatures : \\n', features)\n\n#count frequency of ngrams\nprint('\\n\\nX1 : \\n', ngrams.toarray())\n  \n#apply TFIDF\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(ngram_range = (3, 3))\nngrams = vectorizer.fit_transform(train_df['cleaned_label'])\nscores = (ngrams.toarray())\nprint('\\n\\nScores : \\n', scores)\n  \n#get top ranking features\nsums = ngrams.sum(axis = 0)\ndata1 = []\nfor col, term in enumerate(features):\n    data1.append( (term, sums[0,col] ))\nranking = pd.DataFrame(data1, columns = ['term','rank'])\nwords = (ranking.sort_values('rank', ascending = False))\nprint ('\\n\\nWords head : \\n', words.head(20))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:07.462678Z","iopub.execute_input":"2021-06-05T05:11:07.462983Z","iopub.status.idle":"2021-06-05T05:11:07.920569Z","shell.execute_reply.started":"2021-06-05T05:11:07.462952Z","shell.execute_reply":"2021-06-05T05:11:07.919624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis = 0)\n\n#list of ngrams\nvocab = vectorizer.vocabulary_\ndf_trigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse = True)\n            ).rename(columns = {0: 'frequency', 1:'trigram'})\n\nplt.figure(figsize = (20, 10))\nsns.lineplot(x = df_trigram['trigram'][:60], y = df_trigram['frequency'][:60])\nplt.xticks(rotation = 90, fontsize = 16)\nplt.xlabel('Trigram',fontsize = 20)\nplt.ylabel('Frequency',fontsize = 20)\nplt.title('Cleaned Label Trigram',fontsize = 30)\nplt.show()\n\n#save\nplt.savefig('cleaned_label trigram.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:07.921733Z","iopub.execute_input":"2021-06-05T05:11:07.922037Z","iopub.status.idle":"2021-06-05T05:11:09.650141Z","shell.execute_reply.started":"2021-06-05T05:11:07.922005Z","shell.execute_reply":"2021-06-05T05:11:09.648129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find one-worded, two-worded, three-worded, four-worded, five-worded dataset title\none_worded_cleaned_label = train_df[train_df['cleaned_label'].str.split().apply(len) == 1]\ntwo_worded_cleaned_label = train_df[train_df['cleaned_label'].str.split().apply(len) == 2]\nthree_worded_cleaned_label = train_df[train_df['cleaned_label'].str.split().apply(len) == 3]\nfour_worded_cleaned_label = train_df[train_df['cleaned_label'].str.split().apply(len) == 4]\nfive_worded_cleaned_label = train_df[train_df['cleaned_label'].str.split().apply(len) == 5]\n\n#create a bar plot\nfig, ax = plt.subplots(figsize = (10, 6))\nax.bar([1, 2, 3, 4, 5], [one_worded_cleaned_label.size,\n                         two_worded_cleaned_label.size,\n                         three_worded_cleaned_label.size,\n                         four_worded_cleaned_label.size,\n                         five_worded_cleaned_label.size])\n\n#label the x-axis instances\nax.set_xticks([1, 2, 3, 4, 5])\nax.set_xticklabels([\"one\", \"two\", \"three\", 'four', 'five'])\n\n# set the title and the xy-axis labels\nplt.title(\"Number of Words in Cleaned Label\")\nplt.xlabel(\"Number of Words\")\nplt.ylabel(\"Cleaned Label\")\n\n# display the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:09.651708Z","iopub.execute_input":"2021-06-05T05:11:09.652107Z","iopub.status.idle":"2021-06-05T05:11:10.068152Z","shell.execute_reply.started":"2021-06-05T05:11:09.652069Z","shell.execute_reply":"2021-06-05T05:11:10.067312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SENTIMENT ANALYSIS","metadata":{}},{"cell_type":"markdown","source":"TextBlob is another excellent open-source library for performing NLP tasks with ease, including sentiment analysis. It also an a sentiment lexicon (in the form of an XML file) which it leverages to give both polarity and subjectivity scores. The polarity score is a float within the range [-1.0, 1.0]. The subjectivity is a float within the range [0.0, 1.0] where 0.0 is very objective and 1.0 is very subjective.","metadata":{}},{"cell_type":"code","source":"from textblob import TextBlob\n\n#get texxt sentiment\ntrain_df['text_sentiment'] = train_df['text'].apply(lambda x: TextBlob(x).sentiment.polarity)\n\n#review\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:11:10.069600Z","iopub.execute_input":"2021-06-05T05:11:10.069957Z","iopub.status.idle":"2021-06-05T05:35:44.404392Z","shell.execute_reply.started":"2021-06-05T05:11:10.069920Z","shell.execute_reply":"2021-06-05T05:35:44.403297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from textblob import TextBlob\n\n#get text tag\ntrain_df['dataset_title_tag'] = train_df['dataset_title'].apply(lambda x: TextBlob(x).tags)\n\n#review\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:35:44.406099Z","iopub.execute_input":"2021-06-05T05:35:44.406357Z","iopub.status.idle":"2021-06-05T05:36:05.035015Z","shell.execute_reply.started":"2021-06-05T05:35:44.406332Z","shell.execute_reply":"2021-06-05T05:36:05.033376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\ntrain_df.to_csv('train_df_tag.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:36:05.036421Z","iopub.execute_input":"2021-06-05T05:36:05.036826Z","iopub.status.idle":"2021-06-05T05:36:35.245851Z","shell.execute_reply.started":"2021-06-05T05:36:05.036776Z","shell.execute_reply":"2021-06-05T05:36:35.244626Z"},"trusted":true},"execution_count":null,"outputs":[]}]}