{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Coleridge Starter EDA\n- Identify Most popular/cited Datasets in the Training Data\n- Identify Most important words in a Dataset\n- Identify Top Keywords in the entire Training Corpus\n\n### Addon: Identify a Normal Distribution in the Dataset","metadata":{}},{"cell_type":"markdown","source":"### Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport nltk\nimport spacy\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\nfrom tqdm import tqdm\nfrom functools import partial\nimport re\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom nltk.corpus import stopwords\nfrom spacy.lang.en.stop_words import STOP_WORDS\nimport itertools\nimport collections","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Import Training Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Unique values in the training set:')\nfor col in train_df.columns:\n    print(f'{col} : {train_df[col].nunique()}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Define Paths for Train and Test Json files\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Function to read JSON files and extract publication Text \n\ndef json_to_text(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n            \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = ' '.join(combined)\n    \n    if output=='text':\n        return all_contents\n    elif output=='head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Extract Publication Text for Training Data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(json_to_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Reading the Sample Submission Data\n\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub.head()\n### Extract Publication Text for the sample publications \nsample_sub['text'] = sample_sub['Id'].apply(partial(json_to_text,train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Identify Most popular/cited Datasets in the Training Data","metadata":{}},{"cell_type":"code","source":"#### Set size for sns plots\nsns.set(rc={'figure.figsize':(11.7,8.27)})\n\n## Plot top 15 popular datasets\ntrain_df.dataset_label.value_counts()[:15].plot(kind='bar',title='Famous Datasets',color = sns.color_palette(\"husl\", 8))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- We can look at the percentage distribution of cited Datasets","metadata":{}},{"cell_type":"code","source":"((train_df.dataset_label.value_counts()/train_df.dataset_label.shape[0])*100)[:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Alzheimers Accounts for roughly 30% of entire labels. ","metadata":{}},{"cell_type":"markdown","source":"## Identify Top Keywords in the entire Training Corpus","metadata":{}},{"cell_type":"markdown","source":"#### One of the most Important steps before any keyword identification process is Text Cleaning to avoid GIGO (Garbage In Garbage Out).\n- Lemmatize Text to bring the word to its base form and hence removing redundant words from our vocabulary","metadata":{}},{"cell_type":"code","source":"def lemmatization(text):\n\n    doc = nlp(text)\n    lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n    return ' '.join(lemma_list)\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ntrain_df['text'] = train_df['text'].progress_apply(clean_text)\n\nnlp = spacy.load('en', disable=['parser', 'ner'])\nstop_words = stopwords.words('english')\n\ntry:\n    train_df['text'] = train_df['text'].progress_apply(lemmatization)\nexcept:\n    pass","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\n\ndocs = train_df['text'].tolist()\n\n#Ignore words that appear in 85% texts, \ncv = CountVectorizer(max_df=0.85, stop_words=stop_words, max_features=60000)\nword_count_vector = cv.fit_transform(docs)\n\ntfidf_transformer = TfidfTransformer(smooth_idf=True,use_idf=True)\ntfidf_transformer.fit(word_count_vector)\nfeature_names = cv.get_feature_names()\n\ndef sort_coo(coo_matrix):\n    tuples = zip(coo_matrix.col, coo_matrix.data)\n    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)\n\ndef extract_topn_from_vector(feature_names, sorted_items, topn=10):\n    \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n    \n    #use only topn items from vector\n    sorted_items = sorted_items[:topn]\n\n    score_vals = []\n    feature_vals = []\n    \n    # word index and corresponding tf-idf score\n    for idx, score in sorted_items:\n        \n        #keep track of feature name and its corresponding score\n        score_vals.append(round(score, 3))\n        feature_vals.append(feature_names[idx])\n\n    #create a tuples of feature,score\n    results= {}\n    for idx in range(len(feature_vals)):\n        results[feature_vals[idx]]=score_vals[idx]\n    \n    return results","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nIds = train_df.Id.tolist()\nkeyword_df = pd.DataFrame()\n\nfor i in range(len(docs)):\n    doc = docs[i]\n    Id = Ids[i]\n    tfidf_vector = tfidf_transformer.transform(cv.transform([doc]))\n\n    #sort the tf-idf vectors by descending order of scores\n    sorted_items=sort_coo(tfidf_vector.tocoo())\n\n    #extract only the top n; n here is 10\n    keywords=extract_topn_from_vector(feature_names,sorted_items,10)\n\n    temp_df = pd.DataFrame()\n    temp_df['keyword'] = keywords\n    temp_df['weight'] = keywords.values()\n    temp_df['id']=Id\n    keyword_df = keyword_df.append(temp_df)\n\n### We have a DataFrame with Keywords for each article and its keywords with their weights\nkeyword_df[['id','keyword','weight']].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_df.groupby('keyword')['weight'].sum().sort_values(ascending=False)[:15].plot(\n    kind='bar',title='Keywords with Top Weight',color = sns.color_palette(\"husl\", 8))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Kids and Education seems to be the most common theme -  (Student, School, Children, Teachers in top Keywords)\n- cov and covid are in top keywords, reflects upon the research on the Coronavirus\n- Keywords “Et al.” is short for the Latin term “et alia,” meaning “and others.” It is used in academic citations when referring to a source with multiple authors","metadata":{}},{"cell_type":"code","source":"train_df.dataset_label.value_counts()[:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_df.groupby('keyword')['weight'].sum().sort_values(ascending=False)[:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_df.groupby('keyword')['keyword'].count().sort_values(ascending=False)[:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_df.groupby(['id','keyword'])['weight'].sum().sort_values(ascending=False)[:20]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Identify Keywords for a specific Article","metadata":{}},{"cell_type":"code","source":"### Looking at the keywords from the first article\n\ndoc = docs[0]\ntfidf_vector = tfidf_transformer.transform(cv.transform([doc]))\n\n#sort the tf-idf vectors by descending order of scores\nsorted_items=sort_coo(tfidf_vector.tocoo())\n\n#extract only the top n; n here is 10\nkeywords=extract_topn_from_vector(feature_names,sorted_items,20)\n\nprint(train_df.cleaned_label[0])\n# now print the results\nprint(\"\\n===Keywords===\")\nfor k in keywords:\n    print(k,keywords[k])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How does the Article length Distribution Looks like? Pretty \"Normal\"","metadata":{}},{"cell_type":"code","source":"train_df.text.str.len().plot(kind='hist')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Some Articles are really huge, need to adjust the outliers to get a sense of the distribution","metadata":{}},{"cell_type":"code","source":"## Function to remove outlier article lenghts\n\ndef get_iqr(df):\n    df['text_len'] = df['text'].str.len()\n    sorted_len = np.sort(df['text_len'])\n    Q1,Q3 = np.percentile(sorted_len , [25,75])\n    IQR = Q3-Q1\n    upper_range = Q3 + (1.5 * IQR)\n    return int(upper_range)\n\ndf_eda = train_df.copy()\nupper_limit = get_iqr(df_eda)\nadjusted_len = train_df.text.str.slice(0,upper_limit)\nsns.histplot(data=adjusted_len.str.len())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Publications length follows a right skewed Normal Distribution, with Median around 25000 words, and exceptions with 80K+ words as well.","metadata":{}},{"cell_type":"code","source":"### Save Cleaned Train file and Keywords to csv for quick reference\ntrain_df.to_csv('./train_df_cleaned.csv')\nkeyword_df.to_csv('./keywords.csv')\n#keyword_df = pd.read_csv('../input/coleridgetrainkeywords/keywords.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reference :https://www.kaggle.com/manabendrarout/tabular-data-preparation-basic-eda-and-baseline\n           https://kavita-ganesan.com/extracting-keywords-from-text-tfidf/#.YIQBCpAzaUk\n","metadata":{}}]}