{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1 style='background:teal; color:white; padding:20px;'>\nColeridge: Words in the titles</h1>\n\nref [Coleridge competition @ Kaggle](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data)\n\n## purpose\nillustrate search for frequent words in dataset titles"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\n%matplotlib inline\nfrom IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\")) # full screen width of Jupyter notebook\npd.options.display.max_rows, pd.options.display.max_columns = 500, 100\n\nfrom collections import Counter\nfrom nltk.corpus import stopwords\neng_stopwords = set(stopwords.words('english'))  # 179 englist stopwords\n\ndata_path = '../input/coleridgeinitiative-show-us-the-data/'\ntrain_df = pd.read_csv(os.path.join(data_path, 'train.csv'))\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def titles_to_word_sets(titles_series):\n    # cleanup characters\n    titles_words = titles_series.str.lower().replace(r'[^a-z ]+','', regex=True).unique()\n    # create list of sets of word for each dataset label\n    titles_words = [set(t_words.split()) for t_words in titles_words]\n    # remove stopwords\n    titles_words = [t_words.difference(eng_stopwords) for t_words in titles_words]\n    return titles_words\n\ndef count_occurences (list_of_word_sets):\n    cnt = Counter ([w for word_set in list_of_word_sets for w in word_set])\n    return {k: v for k, v in sorted(cnt.items(), key=lambda item: item[1], reverse=True)}\n\ntitles_words = titles_to_word_sets(train_df.cleaned_label)\noccurencies = count_occurences(titles_words)\nprint(\"Top 10 words by frequency:\")\nlist(occurencies.items())[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_coverage_words(words_sets, min_occurencies=3):\n    \"\"\" finds a few words that covers most word sets provided\n    set min_occurencies to 1 to cover all words set.\n    returns dict with covering words and number of words_sets excluded at iteration \n    \"\"\"\n    coverage_words = {}\n    while len(words_sets) > 0:\n        occur_dict = count_occurences(words_sets)\n        next_word, next_word_count = list(occur_dict.items())[0]\n        if next_word_count < min_occurencies:\n            break\n        words_sets = [word_set for word_set in words_sets if next_word not in word_set]\n        coverage_words[next_word] = next_word_count\n    coverage_words['_REMAINING_'] = len(words_sets)\n    return coverage_words, len(words_sets)\n\ncoverage_words, n_remaining = find_coverage_words(titles_words)\nprint(f\"Analyzing words from {len(titles_words)} titles; found {len(coverage_words)} words\")\ncoverage_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# same test with most frequent words\ntop_n = 11\nfrequent_set = set(list(occurencies.keys())[:top_n])\nlen([tws for tws in titles_words if set(tws).intersection(frequent_set) != set()])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Observations:**\n- With **greedy search** for coverage: at leaset 1 of 11 words are present in **88.5%** of the dataset titles. (115 of 130).\n\n- When taking same number of the **most frequent words**, we observe that only **74.5%** of titles (97 of 130) contain at least one of the 11 most frequent words."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}