{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Kaggle AI. Topic Modeling and Clustering. ","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings('ignore')\n\ntrain_kw = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\", parse_dates=[0, 3])\ntrain_kw_ex = train_kw.copy()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:19:49.312088Z","iopub.execute_input":"2023-07-04T15:19:49.312761Z","iopub.status.idle":"2023-07-04T15:19:49.958846Z","shell.execute_reply.started":"2023-07-04T15:19:49.312721Z","shell.execute_reply":"2023-07-04T15:19:49.957684Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Auxiliar function to remove HTML tags and special characters\n","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"#https://stackoverflow.com/questions/9662346/python-code-to-remove-html-tags-from-a-string\nimport re\nCLEANR = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\\n|\\r|&nbsp;')\n\ndef cleanhtml(row):\n    raw_html = str(row)\n    cleantext = re.sub(CLEANR, '', raw_html)\n    cleantext = re.sub(\" \\d+\", \" \", cleantext) #https://stackoverflow.com/questions/817122/delete-digits-in-python-regex \n    return cleantext\n\ntrain_kw_ex['clean_writeup'] = train_kw['Writeup'].apply(cleanhtml)\ntrain_kw_ex['num_words'] = train_kw_ex['clean_writeup'].str.split().apply(len)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:19:49.960094Z","iopub.execute_input":"2023-07-04T15:19:49.960441Z","iopub.status.idle":"2023-07-04T15:19:50.732656Z","shell.execute_reply.started":"2023-07-04T15:19:49.960414Z","shell.execute_reply":"2023-07-04T15:19:50.731425Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the data","metadata":{}},{"cell_type":"code","source":"train_kw_pr = train_kw_ex.copy()\n#train_kw_pr = train_kw_pr.sample(frac=0.1)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:19:50.734111Z","iopub.execute_input":"2023-07-04T15:19:50.734494Z","iopub.status.idle":"2023-07-04T15:19:50.741227Z","shell.execute_reply.started":"2023-07-04T15:19:50.734464Z","shell.execute_reply":"2023-07-04T15:19:50.740065Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Clean ngrams without information","metadata":{}},{"cell_type":"code","source":"from wordcloud import STOPWORDS\nstopwords = set(STOPWORDS)\n\nfrom nltk.corpus import stopwords as nltk_stopwords\nfrom collections import Counter","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-04T15:19:50.745116Z","iopub.execute_input":"2023-07-04T15:19:50.745457Z","iopub.status.idle":"2023-07-04T15:19:51.602505Z","shell.execute_reply.started":"2023-07-04T15:19:50.745430Z","shell.execute_reply":"2023-07-04T15:19:51.601555Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"custom_stopwords = sorted(list(set(nltk_stopwords.words('english')) | stopwords))\nmy_custom_stopwords = ['kagglers', 'kaggle', 'github', 'public', 'use', 'lb', 'day', 'def', 'thank', 'competition', 'datum', 'self',\n                  'public', 'private', 'return', 'df', 'one', 'https', 'class', 'arxiv', 'congratulation', 'winner', 'code', 'start', 'week', 'init',\n                   'medal', 'etc', 'th', 'teammate'\n                  ]\ncustom_stopwords +=  my_custom_stopwords\n#https://stackoverflow.com/questions/19560498/faster-way-to-remove-stop-words-in-python","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:19:51.603997Z","iopub.execute_input":"2023-07-04T15:19:51.604823Z","iopub.status.idle":"2023-07-04T15:19:51.612730Z","shell.execute_reply.started":"2023-07-04T15:19:51.604785Z","shell.execute_reply":"2023-07-04T15:19:51.611718Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords_dict = Counter(custom_stopwords)\n\ndef cut_longer_text(row):\n    row = re.sub(r'[^\\w\\s]',' ',row)\n    clean_row = ' '.join([word for word in str(row).lower().split() if word not in stopwords_dict])\n    #clean_row = ' '.join( [w for w in clean_row.split() if len(w)>1] )\n    return clean_row\n\ntrain_kw_pr['cut_longer_text'] = train_kw_ex['clean_writeup'].apply(cut_longer_text)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:19:51.614529Z","iopub.execute_input":"2023-07-04T15:19:51.615314Z","iopub.status.idle":"2023-07-04T15:19:52.660138Z","shell.execute_reply.started":"2023-07-04T15:19:51.615257Z","shell.execute_reply":"2023-07-04T15:19:52.659136Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Advanced Tokenization, Stemming, and Lemmatization","metadata":{}},{"cell_type":"code","source":"import spacy\nfrom spacy.tokens import Doc\n\n# regexp used in CountVectorizer\n# (?u) sets unicode flag, i.e. patterns are unicode\n# \\\\b word boundary: the end of a word is indicated by whitespace or a non-alphanumeric character\n# \\\\w alphanumeric: [0-9a-zA-Z_]\n\n#https://github.com/amueller/introduction_to_ml_with_python/issues/152#issuecomment-1348928537\nclass RegexTokenizer:\n    \"\"\"Spacy custom tokenizer\n        Reference https://spacy.io/usage/linguistic-features#custom-tokenizer\n    \"\"\"\n    def __init__(self, vocab, regex_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b'):\n        self.vocab = vocab\n        self.regexp = re.compile(regex_pattern)\n\n    def __call__(self, text):\n        words = self.regexp.findall(text)\n        spaces = [True] * len(words)\n        spaces[-1] = False #no space after last word\n\n        return Doc(self.vocab, words=words, spaces=spaces)\n\nnlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])\nnlp.tokenizer = RegexTokenizer(nlp.vocab)\n\ndef custom_tokenizer(document):\n    doc_spacy = nlp(document)\n    return [token.lemma_ for token in doc_spacy]\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:19:52.661654Z","iopub.execute_input":"2023-07-04T15:19:52.662086Z","iopub.status.idle":"2023-07-04T15:20:00.031894Z","shell.execute_reply.started":"2023-07-04T15:19:52.662030Z","shell.execute_reply":"2023-07-04T15:20:00.030891Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Explore words before to topic Modeling and Writeup Clustering\n## Build a dictionary with lemmatization of words, bigrams, trigrams","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nimport matplotlib.pylab as plt","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:20:00.033444Z","iopub.execute_input":"2023-07-04T15:20:00.034661Z","iopub.status.idle":"2023-07-04T15:20:00.041327Z","shell.execute_reply.started":"2023-07-04T15:20:00.034617Z","shell.execute_reply":"2023-07-04T15:20:00.040307Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Aux functions","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/code/mpwolke/kaggle-writeups-revival?scriptVersionId=132180674&cellId=10\n##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook\nfrom wordcloud import WordCloud\ndef word_cloud_generating(str_word, title, stop_words):\n    wordcloud = WordCloud(background_color = 'green',\n                          colormap= 'Set2',\n                          height =2000,\n                          stopwords=stop_words,\n                          width = 2000\n                         ).generate(str_word)\n    plt.rcParams['figure.figsize'] = (10,10)\n    plt.axis(\"off\")\n    plt.imshow(wordcloud)\n    plt.title(title )\n    plt.show()\n\ndef best_ngrams (tokens, vectorizer):\n    lst_best_bigrams = []\n    for token in tokens:\n        lst_best_bigrams += list(set([w for w in list(vectorizer.vocabulary_.keys()) if token in str(w)]))\n    return lst_best_bigrams\n\ndef drawing_words(ngrams, min_df, max_df, ngram_range, writeup):\n    l = []\n    dict_of_words = pd.Series(index = list(set(ngrams)),data=np.arange(0, len(list(set(ngrams))),1)).to_dict()\n    vect = CountVectorizer(vocabulary=dict_of_words, min_df=min_df, max_df=max_df, ngram_range=(ngram_range, ngram_range), stop_words = custom_stopwords)\n    dtm = vect.fit_transform(writeup)\n    aot = vect.inverse_transform(dtm)\n    \n    for d in aot:\n        for b in d:\n            l.append(''.join(str(b.replace(' ', '_'))))\n    return l\n    \n\ndef ngrams_to_unigrams (ngrams, writeup):\n    writeup_with_ = []\n    ngrams = list(set(ngrams))\n\n    for doc in writeup: \n        str_tmp = ''.join(doc)\n        for t in ngrams:\n            t_without_ = t.replace('_', ' ') \n            str_with_ = str_tmp.replace(t_without_, t)\n            #https://stackoverflow.com/questions/24804453/how-can-i-copy-a-python-string\n            str_tmp = (str_with_ + '.')[:-1]\n        writeup_with_.append(''.join(str_tmp))\n    \n    return writeup_with_","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:20:00.042467Z","iopub.execute_input":"2023-07-04T15:20:00.042772Z","iopub.status.idle":"2023-07-04T15:20:00.060323Z","shell.execute_reply.started":"2023-07-04T15:20:00.042746Z","shell.execute_reply":"2023-07-04T15:20:00.059105Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_writeup_tokens = []\nfor t in train_kw_pr['clean_writeup'].values:\n    clean_writeup_tokens.append(t)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:20:00.061885Z","iopub.execute_input":"2023-07-04T15:20:00.062346Z","iopub.status.idle":"2023-07-04T15:20:00.077859Z","shell.execute_reply.started":"2023-07-04T15:20:00.062286Z","shell.execute_reply":"2023-07-04T15:20:00.077003Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mapping words of Writeup with lemmatization","metadata":{}},{"cell_type":"code","source":"train_kw_pr['tokenizer_writeup'] = train_kw_pr['clean_writeup'].apply(custom_tokenizer)\nclean_tokenizer_writeup = []\nfor d in train_kw_pr['tokenizer_writeup']:\n    clean_tokenizer_writeup.append(' '.join(d))","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:20:00.078905Z","iopub.execute_input":"2023-07-04T15:20:00.079846Z","iopub.status.idle":"2023-07-04T15:22:20.787818Z","shell.execute_reply.started":"2023-07-04T15:20:00.079814Z","shell.execute_reply":"2023-07-04T15:22:20.786599Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Text-processing: clean HTML & tokenizer writeup. The tokenizer is the first component of the processing\n\n![2023-07-02-10-28-04-image.png](attachment:15da1729-903b-4360-ae04-e875cea3537e.png)\n\n[source of image](https://spacy.io/usage/linguistic-features#custom-tokenizer)","metadata":{},"attachments":{"15da1729-903b-4360-ae04-e875cea3537e.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAq0AAACKCAYAAACAanMMAAAgAElEQVR4Ae2993MUybbve/6LEy/uuxEv4t77XrwX9/5w4rg7e/bs2ZsZ/MDgvUc4gfDeexBCCCuc8AKEJCSQECDvEfLee++9hJAErBcrS5ld6q5Wd5e6Wy31qoiM6qq09c2u7E+vylr5T0AbKUAKkAKkAClACpACpAApYOMK/JONt4+aRwqQAqQAKUAKkAKkAClACgBBK30JSAFSgBQgBUgBUoAUIAVsXgGCVpvvImogKUAKkAKkAClACpACpABBK30HSAFSgBQgBUgBUoAUIAVsXgGCVpvvImogKUAKkAKkAClACpACpABBK30HSAFSgBQgBUgBUoAUIAVsXgGCVpvvImogKUAKkAKkAClACpACpABBK30HSAFSgBQgBUgBUoAUIAVsXgGCVpvvImogKUAKkAKkAClACpACpABBK30HSAFSgBQgBUgBUoAUIAVsXgGCVpvvImogKUAKkAKkAClACpACpABBK30HSAFSwK4UGPj+Az43dIJfaTP4lDSNi/C6tBni6zvgy+B3u+orulhSgBQgBeQKELTK1aDPpAApMKEVcM+uhZ9fp8G/eqeMy/DvPqngllENX78RvE7oLypdHClACigqQNCqKAudJAVIgYmkQNvXQVgXUTAuQVUJsJeG5EJtz9eJ1EV0LaQAKUAKGFSAoNWgRJSAFCAFxrsC22KKhgHr5qgiuJBWBZfSx0e4mFYFW2OK4N9kFuI14fnjvVuo/aQAKUAKmKQAQatJclFiUoAUGG8KvClrFsA66U0GPCtqhNj6znEZvEuaYXpglrieZ4UN4607qL2kAClACqhWgKBVtXSUkRQgBcaDAvJpAbdz6sYlrMoh+3FBg4DWBR9zxkMXUBtJAVKAFDCLAgStZpGRCiEFSAFbVABfWOJzQme+yxr3wMrhFWGVX1dz34AtSk9tIgVIAVLA7AoQtJpdUiqQFCAFbEWBhi/9Au5WhuVPGGjdGFUorqu444utyE3tIAVIAVLAogoQtFpUXiqcFCAFxlKBht6JCa2bCFrH8mtFdZMCpMAYKUDQOkbCU7WkAClgeQUIWi2vMdVACpACpIC1FCBotZbSVA8pQApYXQGCVqtLThWSAqQAKWAxBQhaLSYtFUwKkAJjrYC1oPXs3Rdw8tYTFqJr2y0+d5amB4z1N4vqJwVIgbFQgKB1LFSnOkkBUsAqClgLWn+bsRR+mTSXhciqFoJWq/QuVUIKkAL2pgBBq731OF0vKWBHClgLWtEV1fLN+wha7ei7RZdKCpAC1leAoNX6mk+IGr//AOgZ+AbdQ6F38JvidfV//yHSDGImhY2X8eXbd4VYAPS1ydMoFYHneHzfoHIZfbIylFrx7Yemnf162tE7qGmHUkMHZNeKn5U2rlnvgGG9vikUgaf4tX7Rc61yvfC6tLfvsmvFtEobls3r0S0BANvG47GPlTa8RkyD16y0DddLuR28DtReacO+4mmUrrV+lN4DbvmFwOqth1jwScqFlVsOwN+nLITF63aCb3LeMIuqPmjdc/Yay3/0qge4PPWDWYsc4PeZy8Dx0HkIK2sYVgb3w2por296ANdC370kvx+Vuk3+3cB7RmmT30uYXnszz72k6Vdz3EtKYw82Xeil5/slv5cMXav+e0kzTuqqhfeSZuzRX4akh3H3klItpl2r0r2EpWL9ODYMKvS79veAjkkBSyhA0GoJVSdYmUoDftvXQeEnEp2cTwvMVLzqB3n1Il1IVZtOGhykuZN0fWupn0upFGmyWnp0yijv6hPxBxNKdeLxhJNs7fmmL7rO2BMaOkUZ1zKrFctYOOTQ/T98UhXj/WXLhb4qblRM84tfGqtndlC2Yrx7dq1oB86N1N46+zW6I7gobccTy0UZhe26PjwL23tF/ImkcqUiYGOkxg9oZ78udEbVtosybufUKpYxKyibpfnFL10x3qu4UZSBS60qbf/uk8rSLPyYqxQNVzOrRRmfGzp10uS2aq5VjZ/WW/4hMGfpZmZBnfzHMljqsBvmLJOOF652GgaceqH13HWYMmsF4BSCXycvEOCL0wmcjrsMK8MQrPJ4JWjF+5TfSytD83S0wBPOaZp7Kb25WydNdfdXUca++BKdeDyxK65YpME/BdpbUmOXiHdNr9KOZsdLgnNFGqUEAeUtIv55kfK99Hf/dJYGF41Q2u7l1okywmt07yX8o831Wh9RoFQEnEquEGly23p10pR0fBHxRz+X6cTjiS3RRSINjpvaW1xdh4i/mVWjHc2O536QFpP46XWaYrxPSZMow7ekSTHNT77SvTTvg/IqajeyakQZ8fUdOmW0ysb8O3rueZ1MdIIUMLMCBK1mFnQiFZfX1guuGdWgBCX4r5sP+LgnaAUgaB3+7Z8I0IqQePGhN4PWw673GGCGljbA1Nkr2bng4loBnfqgFctYu/0oS383MJKlf59fCdPmrIK/T14AkVWtogwOpYb2BK0ABK2a+83a0KpkgMDfhNLOPk2j6BMpYAEFCFotIOpEKBIfxy4LkawhaB1UeiyFVtS7ObUseBY2KF52WnO3SFPaqWv1w0y8DH3Wtpi6DpGm8YuuZaf966CID6vWteZiHe8rW0UapakMaGHi7VCy2GEZ3iVNLM393DrFa81v6xVl5LTqWoQx08N8SbOXeqxHyY1dooyKLt0fALRM83YGlrcotiOypl2kaVVY4rOlb0DEY1qlDcvm9ShNl8C28Xhss9L2oqiRpcFrVtpQI15GQbuuFQvzoLUM06D2Shv2FS+jpuerThK5FV6NpRXh8cKDVww4Lz/zF3C5budxdu5tZok4NxK0rnE6wtL7JOaI9A57TrJzQXmV4pwhWOXxStCKF8+18CtVtlyjVY+nUbKSolWdxyuBCdbxUXYv4eN17a22R3MvfarXtX5jerQG8nq08+MxPg3g8UpPVzDN46F7SZ8lNrVJcy+V6YEpXsdbPfcSPu3gaZSe0KD1kccrWXOxnUEVmntJacpFpWzsSdRzL+GTG6znQZ7y2INWYN4OJYswtsMjT7qX9D0Fwrp5GVXduvcSTgvg8Y0KT6tQw/kfcgB/C5TGDGwDbaTAaBUgaB2tghM4P//3/tvbDNAHYRP48unSJoAC5ngRa0RozSgWwMmhNbS0XpzjkKkErUvW72LQGlbeqJOe59O31wetE6DL6BLGoQIIqdMDM8XTt7oeXePCOLwsarINKkDQaoOdYktNwqkBSlZWW2ojtYUU0KeANaF1497TDEJxOsHrlHx4EpEoYJRPD/CKz2DnHobEw6+/z2dzZPWB6UjnCVr19TidHwsF0OK8N76EQau+ObVj0S6qc+IpQNA68fpU1RUpv2+qqijKRArYjALWhNb776PZi1bcX+v0uauBLzTAoZXH4X7StMXwPCZVgO1IkKodR9BqM18xaohMgXg900FkSegjKTAqBQhaRyXfxMmM88KU5lBOnCukK7FHBcwBre9yy5nVVD73FOemoiVVeyGBtxnF4PzYF07efALy+ascWl2fv2VxOOUAX8bShlFjjwla7fHbTNdMCpACBK30HYCImnb41T8d8IUrfS8TkEykwHhUwBzQaixIjpSOz2kNKalTDary8glax+O30T7b3Nw3APpeprNPReiqR6MAQeto1JsAedGJ9KX0KjGB3jlN2a/iBLhUugQ7VICg1Q47nS7ZJhTAKWfoJQWNIbviSqBLwdOETTSUGjGuFCBoHVfdZZnG4uCCLo7QJZCSSxbL1EqlkgKWV8BWoNV7aDpBdE2bWSytG6M0iz8Udyi7krO8ulQDKaBfAVzsAhd+4P689blQ018CxZACugoQtOpqQmdIAVJggiiAf8L4j+YfQVlmAUb5Y/qx+owrhPHrUvIfOkG6jy5jnCsQVNEKuNoY+nDVtxzvOL9Ear6VFSBotbLgVB0pQApYVwFcHpgD3r0888wpHStYxXqfFWqWv9W3JKd1FabaSAFSgBSwjgIErdbRmWohBUiBMVLAt1SzLvuUgEzwKm4atxZXv7IWQIsxh3BcFYo2UoAUIAXsRQGCVnvpabpOUsCOFZC/bY/A5xRTDK4Z1XA1s2ZchCsZ1bAzrgT+t2+qANYVoXl23KN06aQAKWCPChC02mOvD12zd3ETLA/Ng5NJ5YBrjtNGCkxUBXCtdIQ8bqEc73tc4538Kk/UbytdFylACuhTgKBVnzJ2cP5+bp34Ee8d/G4HV0yXaM8K4IsgbhnV4js/XsH1YmoluQ+y5y/yOLz26NoO8C9rhncVreOw9dRkW1KAoNWWesPKbQkob4ENkYUsWLlqqo4UGDMFuge+QVRtO7woaoRnhQ3jIuCKdbjwR/vXwTHTjSomBdQqwP8grqQpLWolpHxDChC00leBFCAFSAFSgBQgBSyiAPprJWi1iLR2WShBq112O100KUAKkAKkAClgHQVSmroAQ15br3UqpFomrAIErRO2a+nCSAFSgBQgBUgBUoAUmDgKELROnL6kKyEFSAFSgBQgBUgBUmDCKkDQOmG7li6MFCAFSAFSgBQgBUiBiaMAQevE6UuTr4S8B5gsGWUgBUgBUoAUIAVIgTFSgKB1jIS3hWrv5Gj8tNpCe6gNpAApQAqQAhNPAXoRa+L16Vhd0aigtaWlBRobGyno0aC317bflJRD67cfP8bqO0j1kgKkAClACkxQBcaTy6u+LwPQ1txDQY8GHW1fxvxbajK0lpeXw6NHj2Dnzp3g6OhIwYAG58+fh8jIyDHvaGoAKUAKkAKkAClgbQXGA7SmxJfDo+ux4HwwiIIBDVyPfYRAr3SorWq39leJ1WcStIaEhBCkGoBUfSDv6uoKaJker1ufxzugYJoGo+nrbx1l0JfpQWFIg9q4DVAVs2zEUJ2wEeoKXOwqlBYcg5T8dYohv+EGFDR72G1wrrwGSsGt5il4NMdSUNBgNGPWSHn5yotnUypHSmb1OLQcet7+RKBqAFT1wfznqBKr95nR0BoeHj4MWPfs2QMuLi7g5uZGQY8Gp06dGqbZuXPnoL+/3+qdPNoKEVY7ftpCwUQN+v2iVUvfE+oIHZ4/UfD8CZpf/BukBvwzBQUNgtP+BQLy/0JBS4P9+WvgL/kXKJiowYX696rHrPGWcWDgGzy6Nty6etc5Bp7dTKKgR4PH1z6D24mQYZCfHFtm1a43ClobGhqGwdfDhw+hoKCAghEahIWFweHDh4V+Pj4+Vu1gc1TW/zZOAGv78tNAwbAGCPldS0/Bj54+VV3wJfGSBKyvfoeOt4vtPuQE/HcGrXmRPwMFjQbRqRKwhhbPAwoaDS4WbhLAurTsHlAwrAGH/LRe27KGqhpAjcgUHpgr4OvW2QiI/VAD+Sk9FIzQwO9RttAOrbCtzT1GKG6eJEZBq5+fn4Cuu3fvEqwaAatyqI+Pj4etW7cKDW39BS2lr1aP4xUGrm2rzkJTdAoFAxq0zz3M9Pr6SJ3l4ntHOXR6/cbAtTNsN3TlB9h1qEk8xKA1Pej/grZKd+iqf0Gh/gU01NyEoIJfmaU1ufoI5DfeozCkwdriywxcl5d5QFB3DgUDGswpucX0OlTzWuknYEKdwxeu+CPvS4feQ2J4I8GqEbAqh3pvjwyhYWRQvtW+H0ZB65kzZwRwpaenE7SaCK0IsO7u7kLDzMxMq3XwSBXV9vaz9aDRHYmhbSAiTVhbW254E7QagNaWO35Mr84Z++F7nbq5zH3ptyVr6/O/Qlf6Y7uGVoT2/LBfGbiWJi4iYJVBe0a5E4PW0OI5BKwyaPesvimsrc6NwQStBqD1anO40Cuyq8DQT8K4ji/KbRDA5XUvjYDVRGBFeM2IbxcaPnCLsdr3wSho3b59OwOuI0eOELCqAFaE1tevXwtotRVvAqa6vOo9cIeBWPviEwStBqAVrdFtK88wvfquqZsS8qOvFbr85kjW1mBHu4fWuuSTYl5rc+llAtchcG2p84DgwikMXBOr9hG4ysB1a6krA7F5Je4ErQagFa3RaJXGaQJbK59bDULGoiL0FsAtrcE+xQStKqAVwfXWuUimo+vRD1brRqOg1cnJiQHX0aNHCVpVQqu/v7+AVnypzRY2ObQa057B5AKNtfXycwJXA+Da7PlB0uuvW+Fbvrp5Yl9zPSVrq+dP0JVyx+7BtSB8KgPX4vg/CFpl1tbcir0MWj8WTYe8xjsErkPg6lcjPfJGEDtVH0jgagBc77fGCWvr2/Z0Y34WjEpja94D8OUhDq0hr0sJWlVCq/t5CVpdDqubBmfUl0crEUGrSgiVz1k15rMtQmtyYxfczallQet7offwy9mnkrV1zmFoikwicDUArm2Ol5leX04/0aupoYjudysla2vQWruH1obUi8La2lh0msBVBq4RRbMYuH6q3E7QKrO27iu7wkBsevE18OvMIHA1AK4bK58yvVaUeUDf9wFDw5NR8f/qnQIYVobmGZXe0okIWs3zwhlBq5UA0hjINHcaW4RWNQPDt+Ia6PjrVgZirWcfE7QagNbmt9GStfWnLTCYqG7A7i9+K6ytnZ/d7B5ciyJnM3AtiJlE0CqD1sLKowxagwp/g5z6awSuQ+AaWncbfhly/XSw1o+g1QC0vuxIFtbWpy0Jan4mhuWxxcUFCFoJWmnagAHgnijQiqMRztFEl07t0/dB08d4AlcD4Nq6/xbTq2ev+7DB3JSDnhDJb2vn26V2D63NGTeEtbUu7yCBqwxcY4oXMnCNLd9E0Cqztp4od2Mg9o9CF3jRnkTgagBcd9W8YnrNLr4BjQOdpgxV4yItQStBK0GrHUHrj44e6Jy+T7K2HrtP0GoAWnEaRcfPjkyvgZBkVYP6QGWkxtoaf8HuwbUkehED19yI/yRolUFradUZBq2B+X+FzDoXAtchcE1uuAtTC5wZiO2o9iJoNQCtgd3Z8Nf8i0yvm40RqsYsW85E0ErQStBqR9CKg9HXh0EMwjom7YDmN1EErgbAtfX0Q6ZX9wYX1WN5b9QBCVz95tg9tLZlP4a0wP/KwLU6azuBqwxcP5WsYOAaXbaGoFVmbb1UcY1BGL6U9aA1nsDVALgerXvD9Pp7wSUo+tqoetyyxYwErQStBK12Bq04EHUtOiFZW/e7E7QagFZ0gdU+dQ/Tq983StU4/q05R2NtjT5u9+BaFruKQWt2yP+CztonBK5D4FpTc4VBKy7vmlZ7lsBVBq5ziy4xENtU+Yyg1QC0ogusqUXStIrzdUGqxixbzUTQStBK0DoOoTWgvAW4KxI1g0v/66GXjH52hOZXIQSuBsC1xc2LQWvX4pPwo/uLGsnhy6ezErh6T4Ou7Fd2Da6dea8hI0ha3rUyzYGgVWZtTSpdz8A1onQZQasMWt2rrgtr683mSAJXA+B6sfGj0Cult0LVmGWLmQhaCVoJWschtJrqp1Vp8Ole78xArG3HVYJWA9DKrK0LjjK9cHqFmu17byN0PP8rA9fO8P12Da24SlZF/EZmbc348N+grfI2gesQuDbX3gOc14rW1pSaYwSuMnBdXuzCQGxV+UOCVgPQitZWXJgBp1QcqPZVM2SxPLjqIoa8tl7VZZgzI0ErQavR0JqRkQkurreMDp8Tk4wu29xuqsxZnrm9B6Q2dY/6HpZD67cfP1SVNxCTySAMvQk0Pw0icDUAri0eb5le+CLb95pmVZr3pVyTrK0vf4WujCd2D65ZH/4nA9ey5KUErTJra1qZI4PWsJL5BK0yaH0pW97VpTGEwNUAuKJFGqEVQ0SX6WvMm9vllTHLjhsaWAlaCVqNBsvPn5Pgl0lzjQ4fg8OMLlsNZL70eg3XbtyzaB3YLnND6974EtgVVwKpTV2G7k+Lx/fsvCFZWze7ELQagFa2vOuac0yvPjdvlX3zAzp9pkvW1hAnu4fW6s97GLSmBf4f0FxGy7t2CXB9Dh8KfmPgmli1n8BVBq4bSy4zCFtYeoeg1QC0orV1ZfkDpteWSk+TxyxzQ2tETTvMfZ8NviVNJreFZ7AWtF668BSOHL4Bu3ZdhE2bj8PuXc5w2fk5RAdPjFW47GJxgZSUVHDYvGdYmDJzqYBY7bio6FiLAiXWPX3WcovWYSlo5auM7IorZo9e+A1p7T0uUYqWVgwt994QuBoA12bvUEmvvzjCtzx188S+5j2XrK20vCuD9pzgf2PgWpLwJ1lbBbS+gNyKfQxag4v+oOVdZdD6oVZ65I3Ww9P17whcDYDr47YEYW31b08z+SeGr7zoV6oeNHmlCK38t2/O+2zwUQGv1oLW36dr2EbbWLd5ywmIC60Y10vI2gW0KllD1zjsYNA69Y9lFodHef1x8Qms3vEOrfwG3hlXDMljZHntPSG5dGpbe56g1QC0MmurkxsD195Tj/hYbPK+6+0iydr63sHura11yacYtKYG/DPQ8q4vhoF7aOEMBq6fKneQtVUGrrtKpeVdZxXfoOVdDUArWlu3VHkycF1edh96v/ebPF6ZK4McWvlvn6nwam1onT5rFThsOMzCpKmLhZFu1px1kBbXNG7BlaDVALQmJ6eA53NvuHr9LrhcuQUBgR+GQe67oI9irmx6RuawOLTY8rm0WVnZkJySCtdv3mdfHrS2vg14z0JMbPywfHLIHc1nS0wP4Des9n5nbDEkN1p32sCPlk6NtfWGt1nBNfHVO9i8bif43Hhk1nIRHjGc3X8GXl17aJGyeR06+w/xQq/BhFxV4/lARaiwttLyrgGQF/ozA9fC2MnDoE3zuHw4zNnL+ZLKUwxa3xdOpuVdZdD6uf6OsB4ervUna6sBcPXtTBd6PWn5pGrMMkcmJWjlv4F/vs8GbyMsr1aD1mlLGGNs23ZGgGlMcBnMW7hJgKvrpRciLj/FPHNNrVUOQesI0Br0PgSmyqYRcFP73oOnIDs7m4FmamoaTJu1nH0Z9uw/KeAzNzcXFi7dyM6fOH0J8vLyxBeGl8P3J8+4iHyjgVTtvNaEVn4D74gthiQrwmvfTT8GYu3LTo0IgOcPnoPAu89HTCOHvPDHvqy/bp0xv4eCsuA48V2Q12mNz62H7zC9enbfVD2Wd39YL1lbA1eYbG1N+fAADuzeaXI+fGvfFkNj2hVhbaXlXYcDelTRXAaucRWbydoqA9fT5VcZiP1e6ErLuxqAVrS27qvxZXr9UXwd6gc6VI9bo8k4ErTy3z5D8DqW0IpAGRqYL3531jsc0oHWOzcDwNHxpLDOHjl0Hd77Z+qk43AaH1YJF889hn17L8Pmzcdh/z5XQBiO+lCiNw/PO5o9QaseaEUL64zZK1gnO+08As9f+jKr6NoNu9i5E6c1oPnC67X4Mrz2D2QAeuXqbXZu3mIHyM7OYefu3n/C5tUirP42bRFccr3Jgq9fgNWgte3rIOyOK1EVpgZkink9/EbVt9cHr2iN5XONRjOIyPN2/raTgVjLZWUorQ3/DH/7bR48umT8ggSWhFaEU+/rj+D9/ZdGQ7Q5gbbjb9uYXgPBSXIZjf482JihsbaauLyr65mjMOvPZTYJoGqhuCB8KgPXvMifydoqm9taXX2ZQeu7gr/R8q4yaM1vvAeTCqTlSnfVvCJrqxHg+rcCaYGGG43hRo9TSgnV/vatCss3+rfvz6Bs8C7WnUc71tCKoDhj1irGJX/OcxBgmRhVC0uXbxcMw41pfH/m1H2RlsPmrat+etNPmrJIJz3PZ449QaseaL3ocoN1yoIlG4YBZfzQnFTs0LT0DBG3a98Jln72vDUQGhYpOjTofbBIg5ZQnC6AecdqTitCqz7QtMT57bHFEFun+XdsDpdX2oPR11cRkrV13hFoikgaBoIF7yLB5chFpvmOjXvh5mk3SPX9INKEPvSGe+dvwN1z1yEvIEKcV4JWj4u3AC2v1WEJLF1N+Gd45nqPlfn29jORFwET64nxfAPJ3u/hAeY77QZlH2NFmttnr8HTy3fZcfQzf5Ye88gDB9XKkHh4dOk2i8P28vMVIfHsXOKrIPjo4cWmM2C7eby+feuFp0yvbgdnbSmNPu6NHlre1X++0QAa7HUTFi1aBVOmL4I7rqfh1X0XkTcz7DF433dh5wOeXhXnESQjX9+GXU5O4H75FAR73YD4gHuQGSa53cqLeganj+wH51OHwffBZRaH8RxAi2Kfw323M3DP7QwURHuK8+khj1hdTZl+cPvyKXBy3Ap1qb4inuc3Zt+SeV9YW01Z3tXf+zz4vToPVYWP4I3PBXj++DQUZtwbBr7ZyXfA58U5ePzgJBRn3RdxSTE34M6to+z46pWDsHPXFmiufAaN5U/hyYOTLO7l0zNQX6pZtaul6hm88jzL4oIDXURZOGUBy0qMvgGfo6+D49aN8DFgeLzaaQ2a5V3XWsTamlxyEy6574fIdDd4F+sMVz0OwuU7ByCl9Naw+gJjnFm66w8PQWqZJu5l0Blwf3YE0srdYddhRzhwatuwfAiYlgg3KqUFB37Ov0jLuxoBrafqA5m19W8FzlDY12D0OKWdcGt0kdV+/2YHZcMrGbzaArQuXrqN/RaiEScvuZvB5YH9VwSvoNXU3ysRbrj5wtSZ0hNk5JWXT2IEiKL1lQMtpjl98j6gx4JjRyS2IWg9enQY9Gk/Ald7bOhFrM1b97OOWbnWScxL5fNTJ01dyOLeBWmAFD0U8GkCaEXFTj17/opO2+0JWo8llkNB+3DHzpaAVhyYuuYcYSDWevbxMGj7+OAVLF8iTdNYtGA9A7uIJ68Bra+b1kov4y1euB4mT1sMv01ZCAF3PFl+TIN96D40PcDtuAs7RgBFGMx5Gwbz562BuXNWszJnz14Jh7YfEXVj3gXz18LvUxfBxjXSv9h1K7eKeKxrySIHdux36wkrA+fQ8rRTpy9hcQjYWMf8eWtZmpl/LIdzB86yuNyAcNYmzMcHEa+rD0Qd+qAVz7fP2M/06veO1B7XjTr+3l2rsbYaubwrTguYPE26dxw3bYEzRw8wSERoxPavXLEOli9fyz573bvE4t48cSvG2HwAACAASURBVGPHJw/tY2DKr/PKuaOQH+0Jv09bCBvWbwR3l1MMhjF+9ar1LG/gs6vw29QFsH7tBsD6psxYBEHPr7E4b4/LrNwdW6VBHPPVpviwOGNAVTtNSfQCBq45of9i9PKuCIczZi+GWXOXMlDk1xYVfIUB5cP70h/hlavXwIpVa1h7Y0LdWNxjDylux47N7DzmRSh12i4db96yAf6cvwzWrl/H0pdke8DipSth7oJlrC6MO3p0B4tD2MX8Wxw3wuTpUv8gQKsFVXm+xto7zNpqqeVdI9KkH9x5S5fDnwuXwprN69i17Dm2VcDmKZdd8OvkebBu63pY4bCGpUPIRRjdf3IbTJq2AFZtlL53i1atFPksAavyMv8odGYgtrnSk6ytRoDrjOJrTK+zde8MjlH6XF5ZC1pXhubBx8rWYe20BWhduXqPGC/Q6olusPi4g9MB5JbQD/5ZIs5x6ykRt2nTMXE+PKhQnA97V8DO/2PKQnFOXp65PpOlVY+l9Y+5khmdd6jSPjgkfBiUXrh0XXQmps/MzBoWbyuW1obeflATnGKM+5d6PLEcCrVgld+9tb39zFWWOZw18zJxPxCRJllbZ+6Hpvdxw8Dt3d3nrF/k0wPunJP6ikMeWk0dVjkxyCz5EANyaOX5DzppoBStttNnLIXK0E+srgy/EFbHy6se7Bj7H6G0PFhqy/kD51h83PO3LF4OrXK45HD82OU2S+ew2gn+/HOVsO7Gvwhg5by5/Qw4tGJdQfdeMJBGGJeXp+9zyy1fplfXwhPwo7NHLqXRn78kXZbA1WeG0cu7Ll26Rmd6QGn8S8iOeCqAcdrMxQxEEQz379oBCxauEnEO6zYKKPW8LVnQuQX1rutppk19mi/UJHszYD12YI/Ie/2iBHoIpxxaZ8xaAlh/jqx+bSA15rgj10tYW41d3hWhFfsuN+UOA8RPkdfY8YED29hxZf5DyEzUrLg17Y9F4LBxPYvj0IoQWlP0iJXRMgSfe/Y46gDn3r1bYerMRcLyilZbrPvlszPMQoufMfi+PAdluQ+gqeKpThlyGDXlc2rZJgaukaXLzQ6EEakStG7c4QBZ1bdZ+dsPSOCeXuEO3sHSfXffS7NCF4ItBg6teN1rt6yDjKrbEJN11extlIOq/PMz2YIDtLxrjkFwd20KZdCKLsOSesoNjlP8ySECJN/U/O5hHr/SZqMstCvD8uFj1XBY5XXbArQuWrKV3eczZq9mYHnvViA7xnvgzatkHdhcvXaflH7WKhE3ecYydm7P7kviHAJpUlQds7iiT1hzAapSOQSteqB17kLpH/uR4xd0wFPJupuUlALc9+ukqZKl9dzFqzp5x9rSym8gNXtcXIAPBEp7CVbVrW+vpj3aebo3uEjW1mP3h4Ebh045tG5z2M1uPA6dCHdXT0jWt+AHryByyNKKFk60lq5Z7jisTIROtIailZMHvPGxjMaoZFa24/pdIs8DZ+nRCZaNdSlBa9RTP/jbpLmwf9thkQ/LnD5zmaiDW2JxSgOH1p2b9on0+iBV6Xz7khNMrz4Pw5YLba3Z8fcBjbXVyOVdlaAVoRAfzeNjfQzzF6wETIfnL544BAiWKR8fAj7Sx+kFe3dsZ3Hvn0t/PB7fPA8FMZ5weO9uZnnFfOE+kt6LF69mVla0tK4YsuKyqQhDllb/R1cE1BoDpyOlKY9by8A18+P/bRTwoTX075PnDUuL/b1k+SpxDh/vx0dcY2HB4hWwdIUUx6EVpxjIARKtqVjGqjVrGJDyOATWKTMWMisrwjIHZpxawC2t6zdIQMzzmGvfUf9MZm09bVYo5NB64tJOUe5pV+nejsu9Buev7WV6oIXVYdt6FmbOXQJTZy0aBq0x2daDVTm4rhxa3nU1Le9qEFrxpaxFpZL3hf3VPopDEj+pz9LK403dG3oRC+e8BuuBVV6XLUArPrrH8QHnsCIQXr/izY7xXNi7fB3Y3LFd+tOHLrM4QGJaDCeO3xbneJw19gSteqD18LHzrGOWr96qA55K0Lp91xGWftW67fDkqZf4Inz4GDos/0SE1hNJ5VDUMXawygeF7vXORkMrAiXeeHKYw/mqeA5fkOLQumXdTli1bDMD1/TXwSI9pluxZBNLh2l5wGkDStCK82ExzwcPL1aGNrSiR4FZs1fC0kUbhFUVraaYB62tvHy+x/m3OW+l6QH7th4S7ZJfj6HP7YuOjwpafwz2mQVa3c5Lj5s4YOIUAg6t5Qle8OdczdyqNasdoDjuhQBNnOuKGmGYOXsphL66yeJwj+dwGgGHYb5vzvRj82cx/p2nNF1gJBg1Nq4sdvUQtP4/w0BSH/wZglb3m9KYgrCKkImP7jm0PhqaOvDOz3lYXRV5D8Dz8SlYtUZ63M2trgisy1euFgDMQRjn0KJVFbXA9uhr62jOd9Q9HRNojc2+BmfdJIB94ncS3kReFAHnuMotrQkFNwT0yqHS0p+XD0HrmopHRkEbgps9hwWlt5m1dX+1Lx/2rbLXB60SrLYZ1YaxhlbPhxFirDx4wI0B58O7H8W5J/dDdCB0+Upp6tm8BZtEHLfWYpw1IFW7DoJWPdCKsIkDOYZde49DWHgUZGRkwstXfrDF6SDsP3RawOiLl5J7pF9/nw8RkdHs/BoHqbPnL3aAnJxckfb23UeiXLTOoueAdNkLXUpArPacpV1e2Qqs4ojR//4zA7D22Qeh6aP0yJ5DG0Io9iO+zMTPXRp6OUv+Bv+BbYdZuqw3ocOmB+DxpCkLYcXSTVAfKb3otWaFIwPZUtnLVXmB0otcaqAVraU4rzbLP1S0EduK83BxHitOX+Btx5fL8PNooLXlqhfTq2vpKfjRre4Px5eEcxK0+s6Crhzj5oOitXP2HI33gJL4F0zzCycOCRBFa+qSJavZ8dNbF5j1FB/ha8MjvoyF/er3yFUnrjhOmhJy/MDeYXEVn73YMb70hXnNBa1t2U/F9ICq9I1GwR/OIcU2cCDEx/J4fPzYDvaInn/m8YuXrRRWWH3QytPi/tBBCejx84ZNDvD7tAVQV/JY1Fee94B9tjS0JpeuH5oeYP75osLS6qyxtJ66LP0hRevpQx9pSsgdz6MCSnPq7kJS8U12vO+49Lg0If+6iLc0qPLyH1TdEI+73VuiLQqjj3PCIaA9k9XhWRQD/s1p7LN31Wd4VZVg0brNBdkXGz8KvVJ7K40CRXMl0obW1WH5EFJlHKzyNlgNWodWxHJyOiugMuh1Bsyeu56NLziucFdW6LYKjzHMX7QFkqPrRZ7XL6SFkDDu9Ml74vzOnRdEnnNnHorzCJfpcc3g+/zTsHPa0DnaY4JWPdCKoOh+RwOYvGP5ft6i9YCLCSBwzpwjPZLDOa0cMHFhgb9PXsA697zzNXEeV8TCN/d4ObhHP648nzn3loLWk0nlUGwDllU+GOC+c8oeBmEtl6QXqTjg4Z7PN129fAuc2H0SioKi2PxP1B4f/aOjf/Tjisd7HA8yINT2HvDQ2Z3F4+N/LBOnHLBpAysc2Rv8Z/adYfH4iN9UaEUvAlj35rU7hPeA+xdusnpwcQN53Ik9J9kxXtNooLVjkuQmrP9NrFxGoz9/ay3QWFljzwwDQ224lB8f2isBBVpAn7lfZNMC8PrwZSmET3zhCr0LcEvr26fSi1iYhgd8eQrnrRbGPhcvX/E4tNZyzwLoVQDPnz16gHkK2O64FeYtkHzLmhtaCyNmMmjNjfhPAYVygFT6zB/Ro1UUrarrHKQpSTiPFeES247WUfQwcPbMbvZ4fyRLa23xY5bm9KldzBvA0uWrYOGSFaw9oUGXGbSud1jP4i6elx6bx4Zdtailta7mmrCyZtRdMDsYGoJW9BSwaOVK9rIVThVwvrUPFq5YAftOSF4CxhJaJw+5vXKqfmlRaLwWIb1Uuu3CWUB4xe/Vok1OENiRCf+YthB+m7kEgrqyLdoGc4DrpMLLDFpd6j8aPU6ZKyGH1tXhpsMqb4O1oZWPidp7fNtfDo47d2ggdM78DQxQEXh5PvQQkBLTKPIEeKeIOEwzdeYK5tt17boDgNMIMOD8Vnkd5vxst9CKoOiweQ9s23l4RGCMiooFnCqAc1xnzV3NrK4eDz1FnvsPnrFyDh45K85x8Hzq+YrFbdyyFxKTkkX8w8fPYeEyycqC0w+8fd+IOJ7XHHtzQ+uppHIosTFYxQGh785bycq6/DQDPTmw8s84BxTnnm7bsAcSvAJZuuw3YYAWVzy/feNeeO52n3kVwDxKK2Id3nEUcFoBQi9Pc2zXcZZ/r+NB8L2h8VyAZaKrLV4/vjiF5+JfSC9iYX3Hdp1g8QjCGCcPu7ccEHnx5a1D24+yeJzvyj0cFL2PZueun7oi0vL6Rtq3HrnL9OrZdpWPpybvu4M3MmjtDFhuNLAivKILKoROnGOK4IrnQl7dFPNO3z5xE/CKcTh/9eCeXRD75i48v+0MaJHFgRK9BeBc1sVLVkOU323AfOhGC70JbHLYJNqEc1Z5fU5bHCHh3X0WF+Zzi9UZ9/auSCuHa1M+D1tgIHe/0dCKj+PxWm5cla5p/qLlIH/cH/7BFbZu28SmBuDc1fdvLsHJEztZ+e/8ndn5uPCrw+p77XUedu3awuIOHXKC/LS7Ij7jszucPLGLxe3fvw38vS+wOPQ6gAB9+ZLxbVeCcKVz0cXzGbTGlm8yO7CixTIu5xqbp3rr6RFRPlpVcf7q50LpkT++kIVusBycHNj5HQe3wKchy+qVewfYOXSdxS2g1tifq5AWGPhH4WXwbE+0KDB6FkbDyt174UrwK3hdnwwbjh6FIw9vMVDdeu4MODmft2j95gDW/bWvGbBOLXKDqn7TLJwmD24KGRBaQ6tHV68tQOuFc491YBKto3v3SE+eOKjyPa6iFeCTqpMHpxJM+0My1vG0fL9m7X6IDSnXyWMucLVbaDUHFNp6GeaGVoV72eRT5nZ59b25gwFYx09boMX9tUnwNhLYTdS45qBYoddAbJbJ/YcZBspDNFbWxOE+VU2BPUNp0QMADoQ455WnjfCVLN7ok3XVyvWwbNlaqEx8xeJx/itaaI/u13gM4Pksuc8J/g9mZS2MmyoAUQngtM8pzWnVTjOej0sqTzJgZUu5Nlj/8bs14FNNHQmypVyP1L2xeWA0B3SOpgzvzlQxLeBxS7yqMcsWMlkLWu+7v2Nv8aPvVB5uXn0Nn8KrRgTJj2+ywc3lpciDc2BHAs20uCZAeOV1YN5A37QR84xUnrFxBK0FBRaxctoC0No6tJpjIOk9JFkN2za7ELBGpxjUoG2T5GGh9+Qj1fJ3+f0pWVk/bhYwaSkoROsp/wePe3xJy+X0EVZvjP8d5tZKHr935w7mwspS7dEutybxkJjL2lQ8/E1+Q8A50aE1uHAKg9bPVbutasVUA5LWzONU6sogbE7JLQjoyiJoNfBymUPFE6bXyjIP6Ps+YPS4xVde9CvVXZ3K6ELMmNBa0Gos/I3XdAStBK1mvC0NFxVQ3gIbIgtZMJx65BSDKYXCatj87L1BYJuo1lNjr6v5SZCk1y/b4Fu+uhcZ+jLvCytrV9pDi0Mrh8SEwPvMCwCuYMXP8X1q8EMWx1+y4uetsc98//8yaC1NWmySldUQ0I73+KzyHQxYQ4vnErDKVtXyr7klrIbOjR8JWA0A652WGKFXUIdpT4a4W0a5n9aRf1EsG0vQ2mMWKyxBK0GrZe9UC5bevfYig7DWvTcJWI2wsrYvOMb06rvpp6pXfvS1QcfznyUrq5F+Wa0BjmNVR3mcAwPWtHf/Bdoq3Qla618wDVrrHkBg/s8MWlNqNE79rWnNtNW6lg25uFpFflmNAvb5pe4MWvdUvTJpzDK3n1aTKteTmKCVoHXCPtY319QCW5weoOd+Nvl0v1+0ZDWctBOa30guoIy1ONpjuhbXF0wvXO4W5wGr2XrjjktWVq/fjXZxNVZAael6cQWstMD/k0GrsS6uxrv11Nj2fy5dw4A1snQFWVllVlb3quvCamhpF1ejmUNqK3nPN7wXehmzApb2mIarLmLIaxu+lLh2OmsdE7QStBK0GrAST2Ro7Zp1ULKynnpAVlYjrKzcxdVXr3BVY/RgY4aYFtBpgosrS8PjWJVfHDWPAWtWyP8kC+uQhRWhtrr6MgPWgPy/gCVcXNmqBdWYds0odGYQZmkXV7YCnaNtB3dxdbH+vaoxy9YyEbQStBK02im09l31YcDa/uchaApLJGg1AK2tB28zvbodLqkex3tCHCVo9Z+nM6d0rMBxrOptzrjBgDU14J+hzgQXV8ZaKsdzupjihQxaLeXiyhg4tMU0J8rdhNXQ0i6uRguLtpB/T42P0Ku8v0X1uGVLGQlaCVoJWu0QWr9XNTIAYy6uXF8QsBoA1uaAoWkUP22BgdAUVWN4f/FbjZXVgi6uxgpCTa23IHwyg9b8qF/IyiqzshZWHhFW1hxycSWmRkTVSUuP/iX/ApCLK8PL0Hp1pAhgfdCsbvETVQOdhTMRtBK0ErSOQ2hNbuwC7opEzRjRe9SDQWvbCv0LCdjjnFV919y28RLTq/fgXTVyszxdbxcxaO0MWmP3Vta65OPCymqqi6vxbEE1pu2hhTMYtJKLq3sCWNHqu2PIxRUuJEAurgxD6/ohF1eziq9Dz/evqsctW8tI0ErQStA6DqF1NIsLDMZlaays996QldWAlbXlYYDQazC9WNUY3pdxV1hZreniylTrp7XS54b8J4PW4k+zycoqs7Jmlm9nwIoLCdji4/mxapNXzU1hNSQXV4aB9VZLlNAroD1D1Zhlq5kIWic4tB48eBAcHR1h9+7dBKcG4FSft4Fnz54xDVHHhIQEm7iXRwOtPZtdJSvr5ssErAaAFS2v3MXVl8teqvr+e3cddHr9Q7Kyhm63eytrVYKTsLKSiyvJvRVaYJtq70BQwa8MWsnF1XAr6+riywzCcCEBW5grautt4C6u1pc/VjVm8Uy26PIqK6UanA8GsfD2WZ5ZfJaO1wUCRtNut+MhTMOb58J4d1t8/0/G1HDv3j0BXAEBAQSuKsD1xIkTQsOamhpjZLd4mtrefuaGBF2RmLL1+0QKq2Hzy2CCVgPQ2uLiyfTqnLYPvlerWxHmS8I5jZU1x8euobUz7zVkffj/GLSWp6wiK6vMyppUup4BKy0kMBxYb1aSiytTIPlsQ5Cwsib0lJry86CY1tYWF2iq6xLQeudiNEFriumW1zD/MqGh/7NUxX63xEmjoDUtLU0A1759+yA2NpbA1QRwlUO/q6urJfrRamX++DoAXUOO8Vv33SJgNQCsaGUVLq4eqXMXM1j3WQBrZ/QxuwZWnHpQFruGASsuJGDM/E57SVNRfZEBK7m4Gg6sWQ134c/CSwzCaCEBw9MCEG65i6tDNa/N8tvCV148m6Ju9T+zNEKrkOd3EgR0+TzIIHA1AVw/hzXAtZOhQr+C7HotdS13aBS0YvW3b98W4Lpt2zbw8PCA4OBgCA8Pp6BHA19fXzh37pzQDacG5OfnW643rVBy3+03kpUVFxIIjCFoNQCtwsXVijPw44u6Fxl6wndK0IoLCeQH2HVoybwDaYH/lUFrTfYOglaZlTWueAmDVlpIYDi0nqu4KqyG5OLKMLTKXVyVfFX3ZMgKP0WjrqKiuFlAF04V8LgcBx99iiHuYy0FPRpEv6sC34dZ4HL4vdDu9ZPkUfeFKQUYDa3d3d1w6dKlYQCGEEbBeA2io6NN6RubS/utqBo6/76dQWvr6YcErAaAtdlfM42iPzBeVX/2lwRqrKwJLnYNrAjsxVHzGbDSQgKaeaxoSS6qPCasrOTiSgOtIXXu8GvBRQattJCAYWB93p4kAN+tIUTVmDWeMqUlVAj44nNcaS/N9TVGh6e34qG3p9+qXW40tGKrBgcH4dWrVwSqJsL6+fPnITs726oda4nKvpx5woCVLSQQlUzQagBauYurnp3X1XXH92/Q/W6FBK20kAA0pDozYMWFBBoKT5KVdcjK2lnvCeFFsxi00kICGmBFLwX7yq4ICCMXV4ahlbu4+qXAGTq/fVE3bo2zXCX5jfDoWizB69CLacbAKqYJfZsL3wa/W723TYJW3rr29nb49OkTvHnzBvAROAVlDXD6RGFhIZfN5vamvIg1+ClHmhbw0xZoufKSgNUAsLY8kLm4SshR1fdfsx8JK2tXyl27t7IWhE9h0EoLCQy3smaX7xZW1rFyJ2WL9XrX3BLASgsJGAbWmy2RQq/nrZ9VjVnjOVNlaQt8jiqBiHd5FPRoEPWhALKSq6Gro2/MuloVtI5Za6lisypgisurnh3XGbTSQgIpRgG7cHF15omqPvveVQWdPtMYtNJCAgFQm3RUWFlby6+RlXXIytpY6w4fCiYxaKWFBIZbWTeUSC6ucCEBU96ct9e03MXV/BJ3VWMWZSIFrKEAQas1VLbROuTQOlIT+wPihZW1fflpoGBYA1zaFuf/4jxgNduXJBdhZe18uxg6ZEH7GOO0z2kfy/Ob67MxdWinMXSsr225Af+DQWte5M9AQaNBdOpfGLDiQgKhxfMoDGngXLhJWA2Xlt0DCoY1wKVtMYR25qoZskbMY4veA0ZsMEXarAIErTbbNZZvmLHLuMqhFWGMgnEaoKcFtduXT2cEtHZ4/mTXn5te/KuwsuJ8VgoaDYKy/kNMDUBXVxQkDc7kLxHQymGM9hKUjqTDlopnaoesEfPZmp/WERtLkTatAEGrTXeP7TSuz+MdUDBNg9H03o++VujL9KAwpEHDp21QFbNsxFCd5Ah1BS52F1Ly14FSyKlxhoJmD7sNzpXXQCm4Vt8Hj+ZYCgoatH/rHc2wpZjXFlfEUmwonRwXChC0jotuokaSAqQAKUAKkALjUwFcdRFDXpv5oXh8KkKtVqsAQata5SgfKUAKkAKkAClACpACpIDVFCBotZrUVBEpQAqQAqQAKUAKkAKkgFoFCFrVKjeB8tX19kPXwLcJdEV0KaQAKUAKkAKkACkw0RQgaJ1oPWrC9WS2dMMvfumAb3a+LW8xISclJQVIAVKAFCAFSAFSwLoKELRaV2+bqq244wsDVoTW/Z9Kbapt1BhSgBQgBUiB8a/Akc9l4BRTBM+LGuHLGCz7Of4VpCuQK0DQKlfDzj73Dn6HbTHFcCu7Fqq7v9rZ1dPlkgKkAClAClhSgc7+b/Dz6zRmHFkWkgcd/YOWrI7KtgMFCFrtoJPpEkkBUoAUIAVIAWsr0PRlAI4nljNovZapbnVAa7eZ6rNtBQhabbt/qHWkAClACpACpMC4VqDt6wDU9vSP62ugxtuGAgStttEP1ApSgBQgBUgBUoAUIAVIgREUIGgdQRx7jPIsbADntCp7vHS6ZlKAFCAFSAFSgBSwYQUIWm24c6zdtLPJFcKbwP3cOmtXT/WRAqQAKUAKjHMFTiSVM08B4/wyqPk2qgBBq412zFg0C62s3G8r+nCljRQgBUgBUoAUMFYBBFZ0oYjBu6TJ2GyUjhQwWgGCVqOlso+ETwsaIKqm3T4ulq6SFCAFSAFSwGwKlAz5/v7FLw3wt4Q2UsDcChC0mlvRCVpeVksPPCmoh0ry5zpBe5guixQgBUgB4xTIbe0BHz2W1LDqNgigFRaNE5JSmawAQavJktlnhmOJZeKxDzqM1t4Ofy6DDZGFsD22WDuKHYfXtLN4TBNf36GYZnNUIUtzKrlCMR4HScyPIa+tVycNOq7m8TeyanTi8cTd3DqRpvGLrguW0s4vIh5XcFHaLqRWijRK8UmNXSL+Q2WrUhLY96mUpdkbX6IYH1zVKspIbOxSTMOv9XxqpWL8i6JGUQaufqa9NX4ZEPF3cmq1o9nxrewakabtq65j8IL2XhGv73Hg6eQKlmZTVKFiHfH1naIM/MFT2nbEFrM0hxKUV257V9Eiykhv1p3a0vftu4h3SVd+0fBJQYNIo/TnrLbnq4h/kKc85/tKRrVI0z2ge59kt/aI+DdlzUqXCnivYd9uiylSjI+u1dxLMXXK95JjdBEr41hiuWIZr0ubRTtyWnt00nQNfBPx+vxreuRp7qW6Xt17qaKrT5TxrFDZ6oYvffLv8ddv33XakdqkuZeCKpSXmj6YIN1Lu+KUx57Q6jZRR0JDp04deIK34WyK8tjzqlhzLxW26449rV81Yw8u1qK03c6pFfU09w3oJCnq0Iw9XsXKY8+5FOlewvYqbZ8bNPdScJXyvbQnvoS1Q98qiO8rNWNPSpPu2INLfvMpAGWdfUrNoHOkgMUUIGi1mLQTp2Bcem96YCYbqOa8z1a8sNlB2Sz+b/7pivE4CPOBDgc9pY3HLwnOVYoGt4xqUUaiwo8P/hDwMhBylDb8geNplMAks6VHxF/UA4Orw/JFmn6FH1oETl7Ho/x6pWbAtCE9JwdkKsYjQPEyPuoBXx6/KixfsQzntEpRhhLIVXV/FfEI0UrbrrgSkUYJ8pMbu0Q8ApvStjQkV6RRikerDL8WfX8UfvVPZ2n+eJelVATgi4O8jEiF6S29g99E/IbIAsUyEK55GQjj2ht/9Ilpjn4u045mx1uiCkUZ7QqQj3/YeB364GbuhxyWBlcSUtp8S5pEGX6lyuDL65j3IUepCLieVSPKwD8N2hv+QeFl6INnXJ6Tp1GCl9y2XhGPsKW0rYsoEGlwhT7tDf/E8DoQkpW2me+yWJp/vMlQiobnhZp7SR/48jqWh+YplnE5vUq0QwnkGnr7RfzuOOU/ovviNfcS/gHS3tKau0UZ+v5YrQjNE2m08+NxUIVm7NH3eH7y2wxWBo7pStvD/HpRBwK/9oZjL9dL358R7Tx0TAqYSwGCVnMpOcHLQQukf1kz6APOWUPQii9yKW0vizTQqs/C9O8+0gT+RSqhFS2HfDDVZ/E98EkGrV26VgJ8AY2Xoc+CiZDI06iF1ikB0p8A/AFR2nAqBq9DyVo7+P2HiF+p54cWoZuXoRZad8YWvpwf/gAAAlRJREFUizLwh1l7Q6syr8NVD7TinxCeRjs/HsstN4agFQFFaUMLOq8jQgFaewY00OoQoQytaOHnZShZ8k2FViXLdGydHFqVnwbMfS/9AfyLHmjFJw68na9LlV92+ck3laUxDlp1rbWtsj+AW6OVLb74dIW3o1TB4oaPkHn8GT1PT0yBVn0eTaYHcmhVHnvwBVPeDrTIa28/AET80hB10IqWZl4H/tFT2tDCydPUKEBrapNm7Lmkx+0gQjUv4/sPbPnwzRho/W0IWvGPs9KGTxF4HSEK1lpc8ntteD6cS6kE/D7TRgpYUwGCVmuqTXWRAqQAKUAKkAKkAClACqhSgKBVlWyUiRQgBUgBUoAUIAVIAVLAmgoQtFpTbaqLFCAFSAFSgBQgBUgBUkCVAgStqmSjTKQAKUAKkAKkAClACpAC1lSAoNWaalNdpAApQAqQAqQAKUAKkAKqFCBoVSUbZSIFSAFSgBQgBUgBUoAUsKYCBK3WVJvqIgVIAVKAFCAFSAFSgBRQpQBBqyrZKBMpQAqQAqQAKUAKkAKkgDUVIGi1ptpUFylACpACpAApQAqQAqSAKgUIWlXJRplIAVKAFCAFSAFSgBQgBaypAEGrNdWmukgBUoAUIAVIAVKAFCAFVClA0KpKNspECpACpAApQAqQAqQAKWBNBQharak21UUKkAKkAClACpACpAApoEoBglZVslEmUoAUIAVIAVKAFCAFSAFrKkDQak21qS5SgBQgBUgBUoAUIAVIAVUKELSqko0ykQKkAClACpACpAApQApYU4H/H1SjmUFN326VAAAAAElFTkSuQmCC"}}},{"cell_type":"code","source":"display(clean_writeup_tokens[:1])","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:20.789205Z","iopub.execute_input":"2023-07-04T15:22:20.789608Z","iopub.status.idle":"2023-07-04T15:22:20.798022Z","shell.execute_reply.started":"2023-07-04T15:22:20.789575Z","shell.execute_reply":"2023-07-04T15:22:20.796802Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(clean_tokenizer_writeup[:1])","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:20.803751Z","iopub.execute_input":"2023-07-04T15:22:20.804135Z","iopub.status.idle":"2023-07-04T15:22:20.813169Z","shell.execute_reply.started":"2023-07-04T15:22:20.804079Z","shell.execute_reply":"2023-07-04T15:22:20.812065Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Setup number of ngrams to explore\nPairs of tokens are known as bigrams, triplets of tokens are known as trigrams, and more generally sequences of tokens are known as n-grams.\n\nThe minimum number of tokens should be one, as single words often capture a lot of meaning. Adding bigrams helps in most cases. Adding longer sequences — up to 5-grams— might help too, but this will lead to an explosion of the number of features and might lead to overfitting,\n\n\nCatch <u>all</u> bigrams and trigrams with thresholds","metadata":{}},{"cell_type":"code","source":"vectorizer_bigram_range = CountVectorizer(min_df=0.006, max_df=1.0, ngram_range=(2, 2), stop_words = custom_stopwords )\ntokens_counts_vectorizer = vectorizer_bigram_range.fit_transform(clean_tokenizer_writeup)\n\nvectorizer_trigram_range = CountVectorizer(min_df=0.002, max_df=1.0, ngram_range=(3, 3), stop_words = custom_stopwords )\ntokens_counts_vectorizer = vectorizer_trigram_range.fit_transform(clean_tokenizer_writeup)\n\nprint (f\"Nummber of bigrams: {len(vectorizer_bigram_range.vocabulary_)}, Nummber of trigrams: {len(vectorizer_trigram_range.vocabulary_)}\") ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:22:20.814659Z","iopub.execute_input":"2023-07-04T15:22:20.814979Z","iopub.status.idle":"2023-07-04T15:22:26.698525Z","shell.execute_reply.started":"2023-07-04T15:22:20.814952Z","shell.execute_reply":"2023-07-04T15:22:26.697340Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualize ngrams of Writeup\n`best_words` are frecuent word. ","metadata":{}},{"cell_type":"code","source":"best_words = ['model', 'train', 'score', 'solution', 'first', 'time', 'try',\n       'training', 'work', 'make', 'ensemble', 'find', 'base', 'well',\n       'good', 'feature', 'give', 'prediction', 'add', 'result', 'end',\n       'different', 'cv', 'share', 'think', 'see', 'image', 'improve',\n       'take', 'weight', 'test', 'lot', 'follow', 'two', 'approach',\n        'dataset', 'mean', 'predict', 'last', 'number', 'set',\n       'many', 'much', 'idea', 'simple', 'help', 'fold', 'average',\n       'submission', 'final', 'part', 'notebook', 'host', 'label', 'high',\n       'without', 'post', 'method', 'kernel', 'loss', 'way', 'learn',\n       'size', 'interesting', 'augmentation', 'value', 'top', 'apply',\n       'similar', 'want', 'great', 'large', 'change', 'create', 'epoch',\n       'sample', 'input', 'point', 'small', 'include', 'decide', 'know',\n       'thing', 'run', 'output', 'example', 'data', 'go', 'layer',\n       'problem', 'validation', 'experiment', 'even', 'second', 'select',\n       'single', 'discussion', 'important', 'need', 'seem', 'look',\n       'instead', 'target', 'case', 'low', 'choose', 'remove', 'finally',\n       'inference', 'boost', 'really',  'new', 'increase', 'still',\n       'around', 'generate', 'organizer', 'improvement', 'able',\n       'difference', 'show', 'calculate', 'order', 'random',\n       'big', 'provide', 'read', 'parameter', 'original', 'step',\n       'threshold', 'probability', 'keep', 'reduce', 'another', 'bad',\n       'bit', 'split', 'little', 'information', 'performance', 'due',\n       'next', 'possible', 'less', 'come', 'may', 'nn', 'compare',\n       'almost', 'detail', 'stage', 'hope', 'scale', 'every', 'long',\n       'quite', 'build', 'fast', 'rank', 'previous']","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:26.699925Z","iopub.execute_input":"2023-07-04T15:22:26.700740Z","iopub.status.idle":"2023-07-04T15:22:26.712098Z","shell.execute_reply.started":"2023-07-04T15:22:26.700709Z","shell.execute_reply":"2023-07-04T15:22:26.710810Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Explore write up to find out what unigrams, bigrams, and trigrams are the best one. Recursively generating and drawing give more weight to the ranking of the words than their actual frequencies are high. Explore some unigrams which one are potencial good bigrams, and trigrams.\n\nBigrams and trigrams are captured <u>with thresholds</u> that contain `best_words` are represented","metadata":{}},{"cell_type":"code","source":"lst_bigrams_in = best_ngrams (best_words, vectorizer_bigram_range)\nbigrams_with__ = drawing_words(lst_bigrams_in, 0.006, 1.0, 2, clean_tokenizer_writeup)\n\nlst_bigrams_out = list(set(vectorizer_bigram_range.vocabulary_.keys()) - set(lst_bigrams_in))\nbigrams_with__out = drawing_words(lst_bigrams_out, 0, 1.0, 2, clean_tokenizer_writeup)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:26.713645Z","iopub.execute_input":"2023-07-04T15:22:26.714134Z","iopub.status.idle":"2023-07-04T15:22:31.840647Z","shell.execute_reply.started":"2023-07-04T15:22:26.714105Z","shell.execute_reply":"2023-07-04T15:22:31.839257Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check bigrams and trigrams capatured","metadata":{}},{"cell_type":"code","source":"word_cloud_generating(str(bigrams_with__).replace('\\'', ''), \"Bigrams in vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:31.842895Z","iopub.execute_input":"2023-07-04T15:22:31.843746Z","iopub.status.idle":"2023-07-04T15:22:40.464483Z","shell.execute_reply.started":"2023-07-04T15:22:31.843704Z","shell.execute_reply":"2023-07-04T15:22:40.463388Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_cloud_generating(str(bigrams_with__out).replace('\\'', ''), \"Bigrams out vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:40.465931Z","iopub.execute_input":"2023-07-04T15:22:40.466531Z","iopub.status.idle":"2023-07-04T15:22:47.742719Z","shell.execute_reply.started":"2023-07-04T15:22:40.466492Z","shell.execute_reply":"2023-07-04T15:22:47.741825Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualize Trigrams of Writeup","metadata":{}},{"cell_type":"code","source":"lst_trigrams_in = best_ngrams (best_words, vectorizer_trigram_range)\ntrigrams_with__ = drawing_words(lst_trigrams_in, 0.002, 1.0, 3, clean_tokenizer_writeup)\n\nlst_trigrams_out = list(set(vectorizer_trigram_range.vocabulary_.keys()) - set(lst_trigrams_in))\ntrigrams_with__out = drawing_words(lst_trigrams_out, 0, 1.0, 3, clean_tokenizer_writeup)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:47.744293Z","iopub.execute_input":"2023-07-04T15:22:47.744904Z","iopub.status.idle":"2023-07-04T15:22:52.701993Z","shell.execute_reply.started":"2023-07-04T15:22:47.744870Z","shell.execute_reply":"2023-07-04T15:22:52.700674Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_cloud_generating(str(trigrams_with__).replace('\\'', ''), \"Trigrams in vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:22:52.706058Z","iopub.execute_input":"2023-07-04T15:22:52.706446Z","iopub.status.idle":"2023-07-04T15:23:00.159034Z","shell.execute_reply.started":"2023-07-04T15:22:52.706412Z","shell.execute_reply":"2023-07-04T15:23:00.158222Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_cloud_generating(str(trigrams_with__out).replace('\\'', ''), \"Trigrams out vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:00.160239Z","iopub.execute_input":"2023-07-04T15:23:00.160730Z","iopub.status.idle":"2023-07-04T15:23:04.316996Z","shell.execute_reply.started":"2023-07-04T15:23:00.160701Z","shell.execute_reply":"2023-07-04T15:23:04.314759Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Convert bigrams and trigrams in unigrams in Writeup","metadata":{}},{"cell_type":"code","source":"lst_writeup_with_ = ngrams_to_unigrams (trigrams_with__, clean_tokenizer_writeup)\nlst_writeup_with_ = ngrams_to_unigrams (bigrams_with__, lst_writeup_with_)\n\nprint (f\"Check if works. Trigams spend_much_time: {''.join(lst_writeup_with_).count('spend_much_time')}, Bigram try_feature: {''.join(lst_writeup_with_).count('try_feature')}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:04.318526Z","iopub.execute_input":"2023-07-04T15:23:04.319550Z","iopub.status.idle":"2023-07-04T15:23:37.901016Z","shell.execute_reply.started":"2023-07-04T15:23:04.319511Z","shell.execute_reply":"2023-07-04T15:23:37.899578Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://stackoverflow.com/questions/817122/delete-digits-in-python-regex\ndef delete_digits(doc):\n    doc = re.sub(\" \\d+\", \" \", doc)\n    return doc","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:23:37.902600Z","iopub.execute_input":"2023-07-04T15:23:37.902950Z","iopub.status.idle":"2023-07-04T15:23:37.908658Z","shell.execute_reply.started":"2023-07-04T15:23:37.902922Z","shell.execute_reply":"2023-07-04T15:23:37.907368Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Explore different threshold values of unigrams","metadata":{}},{"cell_type":"code","source":"for md in np.arange(0.01, 0.13, 0.01):\n    vectorizer = CountVectorizer(min_df=md, max_df=0.85, preprocessor = delete_digits,\n                                 stop_words = custom_stopwords )\n    tokens_counts_vectorizer = vectorizer.fit_transform(clean_tokenizer_writeup)\n    print (f\" Nummber of words: {len(vectorizer.vocabulary_)}, min_df {md}\") # vectorizer.vocabulary_","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:23:37.910269Z","iopub.execute_input":"2023-07-04T15:23:37.910919Z","iopub.status.idle":"2023-07-04T15:23:57.889105Z","shell.execute_reply.started":"2023-07-04T15:23:37.910886Z","shell.execute_reply":"2023-07-04T15:23:57.887960Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`custom_stopwords_pr` are normal stop words, `custom_stopwords_ft` added frecuent words","metadata":{}},{"cell_type":"code","source":"custom_stopwords_pr = custom_stopwords.copy\ncustom_stopwords_ft = custom_stopwords + best_words","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:57.890928Z","iopub.execute_input":"2023-07-04T15:23:57.891421Z","iopub.status.idle":"2023-07-04T15:23:57.897155Z","shell.execute_reply.started":"2023-07-04T15:23:57.891364Z","shell.execute_reply":"2023-07-04T15:23:57.896000Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold_unigrams = 0.05","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:57.898653Z","iopub.execute_input":"2023-07-04T15:23:57.899491Z","iopub.status.idle":"2023-07-04T15:23:57.918464Z","shell.execute_reply.started":"2023-07-04T15:23:57.899459Z","shell.execute_reply":"2023-07-04T15:23:57.917118Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Capture <u>all</u> unigrams by setting a threshold without the most frecuent words","metadata":{}},{"cell_type":"code","source":"l_u = []\nvect_u = CountVectorizer(min_df=threshold_unigrams, max_df=0.85, stop_words = custom_stopwords_ft )\ndtm_u = vect_u.fit_transform(clean_tokenizer_writeup)\nlst_threshold_unigrams = list(vect_u.vocabulary_)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:57.920099Z","iopub.execute_input":"2023-07-04T15:23:57.920410Z","iopub.status.idle":"2023-07-04T15:23:59.357316Z","shell.execute_reply.started":"2023-07-04T15:23:57.920384Z","shell.execute_reply":"2023-07-04T15:23:59.356394Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the most frecuent words of unigrams","metadata":{}},{"cell_type":"code","source":"\nvocabulary_in_temp = drawing_words(lst_threshold_unigrams, 0, 1, 1, lst_writeup_with_)\n\ndf_tmp = pd.DataFrame(vocabulary_in_temp, index=np.arange(0, len(vocabulary_in_temp),1))\n#https://stackoverflow.com/questions/47136436/python-pandas-convert-value-counts-output-to-dataframe\ndf_tmp = df_tmp.value_counts().rename_axis('unique_values').reset_index(name='counts')\ndf_tmp[df_tmp.counts > 400]['unique_values'].values","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:23:59.359332Z","iopub.execute_input":"2023-07-04T15:23:59.359757Z","iopub.status.idle":"2023-07-04T15:24:01.557517Z","shell.execute_reply.started":"2023-07-04T15:23:59.359717Z","shell.execute_reply":"2023-07-04T15:24:01.556455Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check bigrams and trigrams capatured without the most frecuent words","metadata":{}},{"cell_type":"code","source":"unigrams_in = drawing_words(lst_threshold_unigrams, threshold_unigrams, 0.85, 1, lst_writeup_with_)\nword_cloud_generating(str(unigrams_in).replace('\\'', ''), \"Unigrams in vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:01.559352Z","iopub.execute_input":"2023-07-04T15:24:01.559766Z","iopub.status.idle":"2023-07-04T15:24:12.117669Z","shell.execute_reply.started":"2023-07-04T15:24:01.559738Z","shell.execute_reply":"2023-07-04T15:24:12.116587Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vect_u = CountVectorizer(min_df=0.01, max_df=0.85, stop_words = custom_stopwords_ft )\ndtm_u = vect_u.fit_transform(clean_tokenizer_writeup)\nlst_threshold_001_unigrams = list(vect_u.vocabulary_)\nlst_unigrams_out = list(set (lst_threshold_001_unigrams) - set (lst_threshold_unigrams))","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:12.119135Z","iopub.execute_input":"2023-07-04T15:24:12.119559Z","iopub.status.idle":"2023-07-04T15:24:13.563567Z","shell.execute_reply.started":"2023-07-04T15:24:12.119521Z","shell.execute_reply":"2023-07-04T15:24:13.562430Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unigrams_out = drawing_words(lst_unigrams_out, 0.01, 0.85, 1, lst_writeup_with_)\nword_cloud_generating(str(unigrams_out).replace('\\'', ''), \"Unigrams out vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:13.564892Z","iopub.execute_input":"2023-07-04T15:24:13.565207Z","iopub.status.idle":"2023-07-04T15:24:25.157073Z","shell.execute_reply.started":"2023-07-04T15:24:13.565182Z","shell.execute_reply":"2023-07-04T15:24:25.156257Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Build vocabulary\nBuild vocabulary and generating and drawing to check more weight to the ranking","metadata":{}},{"cell_type":"code","source":"grams_with__ = []\nfor g in lst_trigrams_in + lst_bigrams_in:\n    grams_with__.append(''.join(str(g).replace(' ', '_')))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:25.158405Z","iopub.execute_input":"2023-07-04T15:24:25.158959Z","iopub.status.idle":"2023-07-04T15:24:25.173657Z","shell.execute_reply.started":"2023-07-04T15:24:25.158927Z","shell.execute_reply":"2023-07-04T15:24:25.172477Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lst_vocabulary_in = grams_with__ + lst_threshold_unigrams\nlst_vocabulary_in = list(set(lst_vocabulary_in))\ncustom_vocabulary = pd.Series(index = lst_vocabulary_in, data=np.arange(0,len(lst_vocabulary_in),1)).to_dict()\nprint (f\"Nummber of trigrams (converted) + bigrams (converted) + the threshold unigrams is: {len(custom_vocabulary.keys())}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:25.175279Z","iopub.execute_input":"2023-07-04T15:24:25.175725Z","iopub.status.idle":"2023-07-04T15:24:25.188580Z","shell.execute_reply.started":"2023-07-04T15:24:25.175694Z","shell.execute_reply":"2023-07-04T15:24:25.187305Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The vocabulary is built, with the bigrams and trigrams that contain the `best_words` and with unigrams set to threshold (`threshold_unigrams`), excluding `best_words`","metadata":{}},{"cell_type":"code","source":"vocabulary_in = drawing_words(lst_vocabulary_in, threshold_unigrams, 0.85, 1, lst_writeup_with_)\nword_cloud_generating(str(vocabulary_in).replace('\\'', ''), \"Unigrams in vocabulary\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:25.190434Z","iopub.execute_input":"2023-07-04T15:24:25.190762Z","iopub.status.idle":"2023-07-04T15:24:36.590104Z","shell.execute_reply.started":"2023-07-04T15:24:25.190734Z","shell.execute_reply":"2023-07-04T15:24:36.588629Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering Writeups","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import silhouette_score\nfrom sklearn.cluster import KMeans\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:36.591757Z","iopub.execute_input":"2023-07-04T15:24:36.592235Z","iopub.status.idle":"2023-07-04T15:24:36.646672Z","shell.execute_reply.started":"2023-07-04T15:24:36.592189Z","shell.execute_reply":"2023-07-04T15:24:36.645388Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vect_tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words = custom_stopwords_ft, vocabulary = custom_vocabulary)\ndtm_tfidf = vect_tfidf.fit_transform(lst_writeup_with_)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:36.649250Z","iopub.execute_input":"2023-07-04T15:24:36.650487Z","iopub.status.idle":"2023-07-04T15:24:37.966649Z","shell.execute_reply.started":"2023-07-04T15:24:36.650435Z","shell.execute_reply":"2023-07-04T15:24:37.965746Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vect_cv = CountVectorizer(vocabulary = custom_vocabulary, stop_words = custom_stopwords_ft)\ndtm_cv = vect_cv.fit_transform(lst_writeup_with_)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:37.968071Z","iopub.execute_input":"2023-07-04T15:24:37.968415Z","iopub.status.idle":"2023-07-04T15:24:39.311315Z","shell.execute_reply.started":"2023-07-04T15:24:37.968386Z","shell.execute_reply":"2023-07-04T15:24:39.310393Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the most frecuent words","metadata":{}},{"cell_type":"code","source":"lst_temp = list(custom_vocabulary.keys())\nvocabulary_in_temp = drawing_words(lst_temp, 0, 1, 1, lst_writeup_with_)\n\ndf_tmp = pd.DataFrame(vocabulary_in_temp, index=np.arange(0, len(vocabulary_in_temp),1))\n#https://stackoverflow.com/questions/47136436/python-pandas-convert-value-counts-output-to-dataframe\ndf_tmp = df_tmp.value_counts().rename_axis('unique_values').reset_index(name='counts')\ndf_tmp[df_tmp.counts > 400]['unique_values'].values","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:39.312755Z","iopub.execute_input":"2023-07-04T15:24:39.313880Z","iopub.status.idle":"2023-07-04T15:24:41.716195Z","shell.execute_reply.started":"2023-07-04T15:24:39.313840Z","shell.execute_reply":"2023-07-04T15:24:41.715166Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_writeups = dtm_cv.toarray()\ntfidf_X_writeups = dtm_tfidf.toarray()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:41.717581Z","iopub.execute_input":"2023-07-04T15:24:41.717896Z","iopub.status.idle":"2023-07-04T15:24:41.934644Z","shell.execute_reply.started":"2023-07-04T15:24:41.717869Z","shell.execute_reply":"2023-07-04T15:24:41.933507Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize high-dimensional data with TSNE\nCollection of text documents to a matrix of token counts","metadata":{}},{"cell_type":"code","source":"from sklearn.manifold import TSNE","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:41.942817Z","iopub.execute_input":"2023-07-04T15:24:41.943179Z","iopub.status.idle":"2023-07-04T15:24:41.948496Z","shell.execute_reply.started":"2023-07-04T15:24:41.943150Z","shell.execute_reply":"2023-07-04T15:24:41.947315Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Matrix of TF-IDF features.","metadata":{}},{"cell_type":"code","source":"n_clusters = 14 # All text of cluters hasn't exceed maximum of 1000000. \nkmeans = KMeans(n_clusters=n_clusters, random_state=42)\nclass_pred = kmeans.fit_predict(tfidf_X_writeups)\n\ntrain_kw_pr['n_cluster'] = class_pred\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:41.949735Z","iopub.execute_input":"2023-07-04T15:24:41.950035Z","iopub.status.idle":"2023-07-04T15:24:51.465411Z","shell.execute_reply.started":"2023-07-04T15:24:41.950010Z","shell.execute_reply":"2023-07-04T15:24:51.464511Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert each collection of text documents of each cluster to a matrix of token counts\n\n## Next code are copied from [COVID-19 Literature Clustering](https://www.kaggle.com/code/maksimeren/covid-19-literature-clustering#Conclusion) by [@maksimeren](https://www.kaggle.com/maksimeren) & [SoloNick](https://www.kaggle.com/solonick)\n\n#### Apply topic Modeling and Document Clustering with LatentDirichletAllocation\n","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import LatentDirichletAllocation","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:51.466701Z","iopub.execute_input":"2023-07-04T15:24:51.468573Z","iopub.status.idle":"2023-07-04T15:24:51.472868Z","shell.execute_reply.started":"2023-07-04T15:24:51.468538Z","shell.execute_reply":"2023-07-04T15:24:51.472052Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topics_per_cluster = 1\nvector_by_cluster = []\nvector_by_topic = []\nvector_by_tfidf = []\ndocument_term_by_cluster = []\ndocument_term_by_topic = []\ndocument_term_by_tfidf = []","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:51.474432Z","iopub.execute_input":"2023-07-04T15:24:51.475046Z","iopub.status.idle":"2023-07-04T15:24:51.484653Z","shell.execute_reply.started":"2023-07-04T15:24:51.475015Z","shell.execute_reply":"2023-07-04T15:24:51.483788Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp = pd.DataFrame(lst_writeup_with_).rename(columns = {0:'lst_writeup_with_'})\ntrain_kw_pr['lst_writeup_with_'] = df_tmp['lst_writeup_with_']","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:51.486173Z","iopub.execute_input":"2023-07-04T15:24:51.486814Z","iopub.status.idle":"2023-07-04T15:24:51.497217Z","shell.execute_reply.started":"2023-07-04T15:24:51.486783Z","shell.execute_reply":"2023-07-04T15:24:51.496419Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ii in range(0, n_clusters):\n    # Creating a vectorizer\n    vector_by_cluster.append(CountVectorizer(stop_words = custom_stopwords_ft,\n                                             #preprocessor = delete_digits,\n                                             vocabulary = custom_vocabulary))","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:51.498652Z","iopub.execute_input":"2023-07-04T15:24:51.499374Z","iopub.status.idle":"2023-07-04T15:24:51.506853Z","shell.execute_reply.started":"2023-07-04T15:24:51.499338Z","shell.execute_reply":"2023-07-04T15:24:51.505945Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for current_cluster, cvec in enumerate(vector_by_cluster):\n    try:\n        document_term_by_cluster.append(cvec.fit_transform(train_kw_pr.loc[train_kw_pr['n_cluster'] == current_cluster,\n                                                                           'lst_writeup_with_']))\n    except Exception as e:\n        print(\"Not enough instances in cluster: \" + str(current_cluster))\n        vectorized_data.append(None)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:51.508286Z","iopub.execute_input":"2023-07-04T15:24:51.508615Z","iopub.status.idle":"2023-07-04T15:24:52.885577Z","shell.execute_reply.started":"2023-07-04T15:24:51.508589Z","shell.execute_reply":"2023-07-04T15:24:52.884612Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Explore words in each cluster","metadata":{}},{"cell_type":"code","source":"lst_topic_id = []\nlst_arrays_of_terms = []\nfor c in np.arange(0,n_clusters,1):\n    \n    arrays_of_terms = vector_by_cluster[c].inverse_transform(document_term_by_cluster[c])\n\n    lst_d =[]\n    for d in arrays_of_terms:\n        d = d.tolist()\n        for w in d:\n            lst_d.append(''.join(w))\n\n    lst_arrays_of_terms.append(str(lst_d).replace('\\'', ''))\n    topic_id = f\"Topic_{c}\"\n    lst_topic_id.append(topic_id)\n\n\ndf_words_by_cluster = pd.DataFrame(lst_arrays_of_terms, index=lst_topic_id)\n#df_words_by_cluster['Topic_id'] = df_words_by_cluster.index\ndf_words_by_cluster.reset_index(inplace=True)\ndf_words_by_cluster.rename(columns = {'index':'Topic_id', 0:'Terms per Topic'}, inplace=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:52.886949Z","iopub.execute_input":"2023-07-04T15:24:52.887226Z","iopub.status.idle":"2023-07-04T15:24:53.761956Z","shell.execute_reply.started":"2023-07-04T15:24:52.887202Z","shell.execute_reply":"2023-07-04T15:24:53.760903Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ii in range(0, n_clusters):\n    lda = LatentDirichletAllocation(n_components = topics_per_cluster,\n                                    max_iter=10, \n                                    learning_method='online',\n                                    verbose=False, \n                                    random_state=42)\n    vector_by_topic.append(lda)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:53.763395Z","iopub.execute_input":"2023-07-04T15:24:53.763779Z","iopub.status.idle":"2023-07-04T15:24:53.770814Z","shell.execute_reply.started":"2023-07-04T15:24:53.763748Z","shell.execute_reply":"2023-07-04T15:24:53.769880Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for current_cluster, lda in enumerate(vector_by_topic):\n    if document_term_by_cluster[current_cluster] != None:\n        document_term_by_topic.append((lda.fit_transform(document_term_by_cluster[current_cluster])))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:53.772221Z","iopub.execute_input":"2023-07-04T15:24:53.772860Z","iopub.status.idle":"2023-07-04T15:24:56.396833Z","shell.execute_reply.started":"2023-07-04T15:24:53.772826Z","shell.execute_reply":"2023-07-04T15:24:56.395703Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for test_cluster in range(n_clusters):\n    lda_tmp = vector_by_topic[test_cluster]\n    sorting = np.argsort(lda_tmp.components_, axis=1)[:, ::-1] \n    feature_names = np.array(vector_by_cluster[test_cluster].get_feature_names_out())\n    '''\n    for idx_word in range(20):\n        print (feature_names[sorting[0, idx_word]])\n    '''","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:56.398139Z","iopub.execute_input":"2023-07-04T15:24:56.398500Z","iopub.status.idle":"2023-07-04T15:24:56.422508Z","shell.execute_reply.started":"2023-07-04T15:24:56.398471Z","shell.execute_reply":"2023-07-04T15:24:56.421527Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check success percent of each cluster's text documents of one cluster.","metadata":{}},{"cell_type":"code","source":"topic_ngram = []\npercent_success = []\ncheck_topic_ngram = []\nidx_cluster = []\n\nfor cluster in range(n_clusters):\n    lda_tmp = vector_by_topic[cluster]\n    sorting = np.argsort(lda_tmp.components_, axis=1)[:, ::-1] \n    feature_names = np.array(vector_by_cluster[cluster].get_feature_names_out())\n    for idx_topic in range (1):\n        #print(f\"Added words of cluster number {cluster}\")\n        for idx_word in range(20):\n            check_topic_ngram.append(feature_names[sorting[idx_topic, idx_word]])\n            topic_ngram.append(feature_names[sorting[idx_topic, idx_word]])\n            idx_cluster.append(cluster)\n        #topic_ngram.append(check_topic_ngram)\n\n    for ngram in check_topic_ngram:\n        ser_train_kw_pr = train_kw_pr.loc[train_kw_pr['n_cluster'] == cluster, 'lst_writeup_with_']\n        bools = ser_train_kw_pr.str.contains(ngram)\n        #print(f\"To word \\\"{ngram}\\\", percent of success {100*(ser_train_kw_pr[bools.values].count()/len(ser_train_kw_pr))} %\")\n        percent_success.append(100*(ser_train_kw_pr[bools.values].count()/len(ser_train_kw_pr)))\n    check_topic_ngram = []","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:56.423898Z","iopub.execute_input":"2023-07-04T15:24:56.424827Z","iopub.status.idle":"2023-07-04T15:24:56.934544Z","shell.execute_reply.started":"2023-07-04T15:24:56.424788Z","shell.execute_reply":"2023-07-04T15:24:56.933389Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_topic_per_success = pd.DataFrame({\"grams\" : topic_ngram,\n                                     \"percent_success\" : percent_success,\n                                    \"cluster_id\": idx_cluster})\n#df_topic_per_success.sort_values(by='percent_success', ascending=False)['grams'].values","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:56.935932Z","iopub.execute_input":"2023-07-04T15:24:56.936253Z","iopub.status.idle":"2023-07-04T15:24:56.942732Z","shell.execute_reply.started":"2023-07-04T15:24:56.936225Z","shell.execute_reply":"2023-07-04T15:24:56.941688Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lst_topic_id = []\nlst_arrays_of_terms =[]\n\nfor c in np.arange(0,n_clusters,1):\n    lst_d = df_topic_per_success[df_topic_per_success.cluster_id==c]['grams'].values\n    str_lst_d = str(lst_d).replace('\\'', '')\n    str_lst_d = str_lst_d.replace('\\n', '')\n    lst_arrays_of_terms.append(str_lst_d)\n    \n    topic_id = f\"Topic_{c}\"\n    lst_topic_id.append(topic_id)\n\n\ndf_topic_by_cluster = pd.DataFrame(lst_arrays_of_terms, index=lst_topic_id)\n#df_words_by_cluster['Topic_id'] = df_words_by_cluster.index\ndf_topic_by_cluster.reset_index(inplace=True)\ndf_topic_by_cluster.rename(columns = {'index':'Topic_id', 0:'Terms per Topic'}, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:56.944116Z","iopub.execute_input":"2023-07-04T15:24:56.944484Z","iopub.status.idle":"2023-07-04T15:24:56.967267Z","shell.execute_reply.started":"2023-07-04T15:24:56.944455Z","shell.execute_reply":"2023-07-04T15:24:56.966375Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize tokens by topic and cluster","metadata":{}},{"cell_type":"code","source":"#https://stackoverflow.com/questions/69997140/multiple-wordcloud-plots-with-specific-shape\nimport seaborn as sns\nsns.set()\n\nwc = WordCloud(background_color=\"white\", colormap=\"Dark2\",\n               max_font_size=150, random_state=42)\n\nplt.rcParams['figure.figsize'] = [20, 15]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:24:56.969208Z","iopub.execute_input":"2023-07-04T15:24:56.970133Z","iopub.status.idle":"2023-07-04T15:24:57.096989Z","shell.execute_reply.started":"2023-07-04T15:24:56.970089Z","shell.execute_reply":"2023-07-04T15:24:57.095794Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tsne_model = TSNE(perplexity=135, n_components=2, init='pca', n_iter=1500, random_state=23)\nX_tsne = tsne_model.fit_transform(tfidf_X_writeups)\n#temp = pd.DataFrame(df_tsne, columns=['tsne1', 'tsne2'])","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:24:57.098541Z","iopub.execute_input":"2023-07-04T15:24:57.099192Z","iopub.status.idle":"2023-07-04T15:25:39.531979Z","shell.execute_reply.started":"2023-07-04T15:24:57.099153Z","shell.execute_reply":"2023-07-04T15:25:39.531055Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C = [] #Color\nL = [] #Label\nfor i in np.arange(0, n_clusters, 1):\n    c = f\"C{i}\"\n    l = f\"Topic {i}\"\n    C.append(c)\n    L.append(l)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:25:39.533531Z","iopub.execute_input":"2023-07-04T15:25:39.534236Z","iopub.status.idle":"2023-07-04T15:25:39.540330Z","shell.execute_reply.started":"2023-07-04T15:25:39.534200Z","shell.execute_reply":"2023-07-04T15:25:39.539436Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_y = np.c_[X_tsne ,class_pred]","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:25:39.543983Z","iopub.execute_input":"2023-07-04T15:25:39.546604Z","iopub.status.idle":"2023-07-04T15:25:39.557584Z","shell.execute_reply.started":"2023-07-04T15:25:39.546555Z","shell.execute_reply":"2023-07-04T15:25:39.556459Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.colors import ListedColormap\n\nC = ['aliceblue', 'yellow', 'aqua', 'aquamarine', 'darkorange', 'beige', 'darkslategray', \n     'black', 'darkseagreen', 'blue', 'blueviolet', 'brown', 'burlywood', 'cadetblue']\n\nfig = plt.figure()\ncolors = ListedColormap(C)\nax = plt.scatter(X_y[:,0], X_y[:,1], c=X_y[:,2], cmap=colors)\n\nplt.legend(handles=ax.legend_elements()[0], labels=L)\n\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:25:39.559063Z","iopub.execute_input":"2023-07-04T15:25:39.560706Z","iopub.status.idle":"2023-07-04T15:25:40.690417Z","shell.execute_reply.started":"2023-07-04T15:25:39.560662Z","shell.execute_reply":"2023-07-04T15:25:40.689506Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = train_kw_pr['n_cluster'].plot.hist(bins=2*n_clusters, title='Nums of writeup per cluster', figsize=(8, 4))\nax.set_xlabel('Id of cluster')\nax.set_ylabel('Nums. of Writeups')\nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:25:40.691726Z","iopub.execute_input":"2023-07-04T15:25:40.692536Z","iopub.status.idle":"2023-07-04T15:25:41.154685Z","shell.execute_reply.started":"2023-07-04T15:25:40.692503Z","shell.execute_reply":"2023-07-04T15:25:41.153434Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots for each topic\nfor i in range(n_clusters):\n    wc.generate(text=df_words_by_cluster[\"Terms per Topic\"][i])   \n    plt.subplot(5, 4, i+1)\n    plt.imshow(wc, interpolation=\"bilinear\")\n    plt.axis(\"off\")\n    plt.title(f'Tokens by cluster {i}')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-07-04T15:25:41.155994Z","iopub.execute_input":"2023-07-04T15:25:41.156365Z","iopub.status.idle":"2023-07-04T15:25:48.575929Z","shell.execute_reply.started":"2023-07-04T15:25:41.156334Z","shell.execute_reply":"2023-07-04T15:25:48.574372Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Create subplots for each topic\nfor i in range(n_clusters):\n    wc.generate(text=df_topic_by_cluster[\"Terms per Topic\"][i])   \n    plt.subplot(5, 4, i+1)\n    plt.imshow(wc, interpolation=\"bilinear\")\n    plt.axis(\"off\")\n    plt.title(f'Topic by cluster {i}')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:25:48.577674Z","iopub.execute_input":"2023-07-04T15:25:48.578385Z","iopub.status.idle":"2023-07-04T15:25:52.442738Z","shell.execute_reply.started":"2023-07-04T15:25:48.578339Z","shell.execute_reply":"2023-07-04T15:25:52.441532Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}