{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 230602_2023 Kaggle AI Report Transformers as Feature Extractors\nThis work tries to process and classify with transformers part of the data written by the participants who obtained good results in the Kaggle contests.\n\nThe experiment has been unsuccessful, since when using a classifier, processing and without processing stop words, the results have been very poor.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-10T12:28:46.416928Z","iopub.execute_input":"2023-06-10T12:28:46.417335Z","iopub.status.idle":"2023-06-10T12:28:46.457293Z","shell.execute_reply.started":"2023-06-10T12:28:46.417307Z","shell.execute_reply":"2023-06-10T12:28:46.455837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get the data","metadata":{}},{"cell_type":"code","source":"train_kw = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\", parse_dates=[0, 3])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:46.470532Z","iopub.execute_input":"2023-06-10T12:28:46.471037Z","iopub.status.idle":"2023-06-10T12:28:47.384217Z","shell.execute_reply.started":"2023-06-10T12:28:46.471003Z","shell.execute_reply":"2023-06-10T12:28:47.382959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```train_arxiv = pd.read_json(\"/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json\", \n                              lines=True,\n                             convert_dates=True)```","metadata":{"execution":{"iopub.status.busy":"2023-06-05T16:20:38.970642Z","iopub.execute_input":"2023-06-05T16:20:38.971134Z","iopub.status.idle":"2023-06-05T16:21:42.349803Z","shell.execute_reply.started":"2023-06-05T16:20:38.971087Z","shell.execute_reply":"2023-06-05T16:21:42.345912Z"}}},{"cell_type":"markdown","source":"# Explore the data","metadata":{}},{"cell_type":"code","source":"train_kw_ex = train_kw.copy()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.388532Z","iopub.execute_input":"2023-06-10T12:28:47.389521Z","iopub.status.idle":"2023-06-10T12:28:47.396507Z","shell.execute_reply.started":"2023-06-10T12:28:47.389473Z","shell.execute_reply":"2023-06-10T12:28:47.394903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.398794Z","iopub.execute_input":"2023-06-10T12:28:47.399331Z","iopub.status.idle":"2023-06-10T12:28:47.436719Z","shell.execute_reply.started":"2023-06-10T12:28:47.399291Z","shell.execute_reply":"2023-06-10T12:28:47.435556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex['Title of Competition'].unique().shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.442361Z","iopub.execute_input":"2023-06-10T12:28:47.442835Z","iopub.status.idle":"2023-06-10T12:28:47.451232Z","shell.execute_reply.started":"2023-06-10T12:28:47.442792Z","shell.execute_reply":"2023-06-10T12:28:47.449922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Study each attribute and its characteristics:  Type \n##### Kaggle Competition Writeups","metadata":{}},{"cell_type":"code","source":"train_kw.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.452767Z","iopub.execute_input":"2023-06-10T12:28:47.453170Z","iopub.status.idle":"2023-06-10T12:28:47.482056Z","shell.execute_reply.started":"2023-06-10T12:28:47.453140Z","shell.execute_reply":"2023-06-10T12:28:47.481096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Metadata for recent arXiv publications\n\n[Source](https://www.kaggle.com/datasets/Cornell-University/arxiv)\n\n\n- `id`: ArXiv ID (can be used to access the paper, see below)\n- `submitter`: Who submitted the paper\n- `authors`: Authors of the paper\n- `title`: Title of the paper\n- `comments`: Additional info, such as number of pages and figures\n- `journal-ref`: Information about the journal the paper was published in\n- `doi`: [[https://www.doi.org](Digital](https://www.doi.org]%28digital/) Object Identifier)\n- `abstract`: The abstract of the paper\n- `categories`: Categories / tags in the ArXiv system\n- `versions`: A version history","metadata":{}},{"cell_type":"code","source":"# train_arxiv.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.483475Z","iopub.execute_input":"2023-06-10T12:28:47.484142Z","iopub.status.idle":"2023-06-10T12:28:47.496662Z","shell.execute_reply.started":"2023-06-10T12:28:47.484102Z","shell.execute_reply":"2023-06-10T12:28:47.495477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Identify promising transformations. \n#### Remove HTML tags and special characters","metadata":{}},{"cell_type":"code","source":"#https://stackoverflow.com/questions/9662346/python-code-to-remove-html-tags-from-a-string\nimport re\nCLEANR = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\\n|\\r')\n#CLEANR = re.compile('<.*?>') \ndef cleanhtml(row):\n    raw_html = str(row)\n    cleantext = re.sub(CLEANR, '', raw_html)\n    return cleantext\n\ntrain_kw_ex['clean_writeup'] = train_kw['Writeup'].apply(cleanhtml)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.498056Z","iopub.execute_input":"2023-06-10T12:28:47.499399Z","iopub.status.idle":"2023-06-10T12:28:47.934465Z","shell.execute_reply.started":"2023-06-10T12:28:47.499310Z","shell.execute_reply":"2023-06-10T12:28:47.933242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex['num_words'] = train_kw_ex['clean_writeup'].str.split().apply(len)\ntrain_kw_ex['num_words'].nlargest(n=5)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:47.935820Z","iopub.execute_input":"2023-06-10T12:28:47.936175Z","iopub.status.idle":"2023-06-10T12:28:48.185977Z","shell.execute_reply.started":"2023-06-10T12:28:47.936146Z","shell.execute_reply":"2023-06-10T12:28:48.184567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex['num_words'].nsmallest(n=5)\ntrain_kw_ex[train_kw_ex.num_words==1]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:48.187331Z","iopub.execute_input":"2023-06-10T12:28:48.187661Z","iopub.status.idle":"2023-06-10T12:28:48.213387Z","shell.execute_reply.started":"2023-06-10T12:28:48.187635Z","shell.execute_reply":"2023-06-10T12:28:48.212078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the data.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pylab as plt\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\nstopwords = set(STOPWORDS)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:48.884630Z","iopub.execute_input":"2023-06-10T12:28:48.885004Z","iopub.status.idle":"2023-06-10T12:28:48.915146Z","shell.execute_reply.started":"2023-06-10T12:28:48.884972Z","shell.execute_reply":"2023-06-10T12:28:48.913666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/mpwolke/kaggle-writeups-revival?scriptVersionId=132180674&cellId=10\n##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook\n\ndef word_cloud_generating(str_word, title, stop_words):\n    wordcloud = WordCloud(background_color = 'green',\n                          colormap= 'Set2',\n                          height =2000,\n                          stopwords=stop_words,\n                          width = 2000\n                         ).generate(str_word)\n    plt.rcParams['figure.figsize'] = (10,10)\n    plt.axis(\"off\")\n    plt.imshow(wordcloud)\n    plt.title(title )\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:48.916597Z","iopub.execute_input":"2023-06-10T12:28:48.916979Z","iopub.status.idle":"2023-06-10T12:28:48.924381Z","shell.execute_reply.started":"2023-06-10T12:28:48.916945Z","shell.execute_reply":"2023-06-10T12:28:48.923068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex['num_words'].hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:48.218282Z","iopub.execute_input":"2023-06-10T12:28:48.218713Z","iopub.status.idle":"2023-06-10T12:28:48.883290Z","shell.execute_reply.started":"2023-06-10T12:28:48.218682Z","shell.execute_reply":"2023-06-10T12:28:48.881919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_cloud_generating(str(train_kw_ex['clean_writeup']), \"ngrams\", stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:48.925921Z","iopub.execute_input":"2023-06-10T12:28:48.926286Z","iopub.status.idle":"2023-06-10T12:28:51.938328Z","shell.execute_reply.started":"2023-06-10T12:28:48.926234Z","shell.execute_reply":"2023-06-10T12:28:51.937443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Descriptive statistics\n#### Word generating and drawing\n#### Counting n-gram frequency","metadata":{}},{"cell_type":"code","source":"#https://stackoverflow.com/questions/14364762/counting-n-gram-frequency-in-python-nltk\nimport nltk\nfrom nltk.util import ngrams\nfrom nltk.collocations import BigramCollocationFinder\nfrom nltk.metrics import BigramAssocMeasures","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:51.939494Z","iopub.execute_input":"2023-06-10T12:28:51.940499Z","iopub.status.idle":"2023-06-10T12:28:53.153089Z","shell.execute_reply.started":"2023-06-10T12:28:51.940434Z","shell.execute_reply":"2023-06-10T12:28:53.151724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Frequency distribution  the number of times each word, bigram o trigram has occurred.","metadata":{}},{"cell_type":"code","source":"clean_writeup_tokens = []\nfor t in train_kw_ex['clean_writeup'].values:\n    clean_writeup_tokens.append(t)\n\nword_fd = nltk.FreqDist(str(clean_writeup_tokens).lower().split())\nbigram_fd = nltk.FreqDist(nltk.bigrams(str(clean_writeup_tokens).lower().split()))\ntrigram_fd = nltk.FreqDist(nltk.trigrams(str(clean_writeup_tokens).lower().split()))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:28:53.154751Z","iopub.execute_input":"2023-06-10T12:28:53.155237Z","iopub.status.idle":"2023-06-10T12:29:02.167935Z","shell.execute_reply.started":"2023-06-10T12:28:53.155192Z","shell.execute_reply":"2023-06-10T12:29:02.166908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freqDist_ngrams (str_of_tokens):\n    word_fd = nltk.FreqDist(str_of_tokens.lower().split())\n    bigram_fd = nltk.FreqDist(nltk.bigrams(str_of_tokens.lower().split()))\n    trigram_fd = nltk.FreqDist(nltk.trigrams(str_of_tokens.lower().split()))\n    \n    return word_fd, bigram_fd, trigram_fd\n","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:02.169289Z","iopub.execute_input":"2023-06-10T12:29:02.170578Z","iopub.status.idle":"2023-06-10T12:29:02.178167Z","shell.execute_reply.started":"2023-06-10T12:29:02.170527Z","shell.execute_reply":"2023-06-10T12:29:02.176728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"List the n most common words, bigram and trigrams and their counts. In raw text there are 482 bigrams and 178 trigrams","metadata":{}},{"cell_type":"code","source":"bigram_fd.most_common()[:15] #Length: 482\ntrigram_fd.most_common()[:15] #Length: 178","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:02.180208Z","iopub.execute_input":"2023-06-10T12:29:02.181155Z","iopub.status.idle":"2023-06-10T12:29:02.799498Z","shell.execute_reply.started":"2023-06-10T12:29:02.181109Z","shell.execute_reply":"2023-06-10T12:29:02.798091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Make some conversions of format\nConvert to ´df´, ´list´ and ´serial´ to extract **ONLY WORDS**","metadata":{}},{"cell_type":"code","source":"df_word_fd = pd.DataFrame(word_fd.items(), columns=['word', 'n_times'])\ndf_word_fd.sort_values(by='n_times', ascending=False, inplace=True)\nlist_word_fd = df_word_fd.sort_values(by='n_times', ascending=False)['word'].values \n\ndf_bigram_fd = pd.DataFrame(bigram_fd.items(), columns=['bigram', 'n_times'])\ndf_bigram_fd.sort_values(by='n_times', ascending=False, inplace=True)\nlist_bigram_fd = df_bigram_fd.sort_values(by='n_times', ascending=False)['bigram'].values \n\ndf_trigram_fd = pd.DataFrame(trigram_fd.items(), columns=['trigram', 'n_times'])\ndf_trigram_fd.sort_values(by='n_times', ascending=False, inplace=True)\nlist_trigram_fd = df_trigram_fd.sort_values(by='n_times', ascending=False)['trigram'].values ","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:02.801304Z","iopub.execute_input":"2023-06-10T12:29:02.801769Z","iopub.status.idle":"2023-06-10T12:29:04.856396Z","shell.execute_reply.started":"2023-06-10T12:29:02.801729Z","shell.execute_reply":"2023-06-10T12:29:04.855072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ser_count_bigrams = df_bigram_fd.sort_values(by='n_times', ascending=False)['n_times'].value_counts() # Length: 482\ndf_count_bigrams = pd.DataFrame(ser_count_bigrams.index, ser_count_bigrams.values)\ndf_count_bigrams.reset_index(inplace=True)\ndf_count_bigrams.rename(columns = {'index':'num_ngrams', 0: \"matched\" }, inplace=True)\n\nser_count_trigrams = df_trigram_fd.sort_values(by='n_times', ascending=False)['n_times'].value_counts() #Length: 178\ndf_count_trigrams = pd.DataFrame(ser_count_trigrams.index, ser_count_trigrams.values)\ndf_count_trigrams.reset_index(inplace=True)\ndf_count_trigrams.rename(columns = {'index':'num_ngrams', 0: \"matched\" }, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:04.858104Z","iopub.execute_input":"2023-06-10T12:29:04.858445Z","iopub.status.idle":"2023-06-10T12:29:05.038995Z","shell.execute_reply.started":"2023-06-10T12:29:04.858417Z","shell.execute_reply":"2023-06-10T12:29:05.037437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Inspect some output of object\nFilter some range","metadata":{}},{"cell_type":"code","source":"df_temp = df_trigram_fd[(df_trigram_fd.n_times<15) & (df_trigram_fd.n_times>8)]\ndf_temp.sort_values(by='n_times', ascending=False, inplace=True)\ndf_temp","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.040742Z","iopub.execute_input":"2023-06-10T12:29:05.041154Z","iopub.status.idle":"2023-06-10T12:29:05.068346Z","shell.execute_reply.started":"2023-06-10T12:29:05.041125Z","shell.execute_reply":"2023-06-10T12:29:05.066841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_count_trigrams[df_count_trigrams.matched>14]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.070297Z","iopub.execute_input":"2023-06-10T12:29:05.070714Z","iopub.status.idle":"2023-06-10T12:29:05.088460Z","shell.execute_reply.started":"2023-06-10T12:29:05.070684Z","shell.execute_reply":"2023-06-10T12:29:05.087078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(list_word_fd[:15])\ndisplay(list_bigram_fd[:15])\ndisplay(list_trigram_fd[:15])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.089819Z","iopub.execute_input":"2023-06-10T12:29:05.090157Z","iopub.status.idle":"2023-06-10T12:29:05.109423Z","shell.execute_reply.started":"2023-06-10T12:29:05.090130Z","shell.execute_reply":"2023-06-10T12:29:05.107899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Count number of times with each n-gram was matched","metadata":{}},{"cell_type":"code","source":"ser_count_bigrams.describe(percentiles=[.70, .80, .90])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.110796Z","iopub.execute_input":"2023-06-10T12:29:05.111129Z","iopub.status.idle":"2023-06-10T12:29:05.128216Z","shell.execute_reply.started":"2023-06-10T12:29:05.111103Z","shell.execute_reply":"2023-06-10T12:29:05.126527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 542446 bigrams was matched only one time  \nThere are 59635 bigrams was matched twice times  \nThere are 21336 bigrams was matched three times \n\nThere are one bigram was matched 6990 times, other was matched 420 times, another was matched 417 times, ...\n\n```\n1       542446\n2        59635\n3        21336\n4        11234\n5         6699\n         ...  \n408          1\n413          1\n417          1\n420          1\n6990         1\n\n```\n","metadata":{}},{"cell_type":"code","source":"\nser_count_trigrams.describe(percentiles=[.30, .35, .40, .45, .55, .60, .65, .70, .75])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.131307Z","iopub.execute_input":"2023-06-10T12:29:05.131769Z","iopub.status.idle":"2023-06-10T12:29:05.147008Z","shell.execute_reply.started":"2023-06-10T12:29:05.131738Z","shell.execute_reply":"2023-06-10T12:29:05.145416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the data. ","metadata":{}},{"cell_type":"code","source":"train_kw_pr = train_kw_ex.copy()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:29:05.149084Z","iopub.execute_input":"2023-06-10T12:29:05.149465Z","iopub.status.idle":"2023-06-10T12:29:05.160373Z","shell.execute_reply.started":"2023-06-10T12:29:05.149436Z","shell.execute_reply":"2023-06-10T12:29:05.158641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data clean\n### Remove outliers\n#### Find out relevant ngrams\nCopy and Paste 1.A4.a1.b2 Counting n-gram frequency","metadata":{}},{"cell_type":"markdown","source":"##### Is stopwords removal ,Stemming and Lemmatization necessary for text classification while using Spacy,Bert or other advanced NLP models for getting the vector embedding of the text?\n\nYou can test to see if doing stemming lemmatization and stopword removal helps. It doesn't always. I usually do if I gonna graph as the stopwords clutter up the results.\n\n**A case for not using Stopwords** Using Stopwords will provide context to the user's intent, so when you use a contextual model like BERT. In such models like BERT, all stopwords are kept to provide enough context information like the negation words (not, nor, never) which are considered to be stopwords.\n\nAccording to https://arxiv.org/pdf/1904.07531.pdf\n\n\"Surprisingly, the stopwords received as much attention as non-stop words, but removing them has no effect inMRR performances. \"\n\n[Source](https://stackoverflow.com/questions/63633534/is-it-necessary-to-do-stopwords-removal-stemming-lemmatization-for-text-classif)\n\n#### Clean ngrams which no information","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords as nltk_stopwords\nfrom collections import Counter\n\ncustom_stopwords = sorted(list(set(nltk_stopwords.words('english')) | stopwords))\nmy_custom_words = ['kagglers', '---', 'nan,', '[0', '[1', '0]', '1]', '0,', '1,', '2,', '3,', '4,', '5,', '6,', '7,', '8,', '9,', 'thank', 'thanks', 'competition', '1,', '1],', '[1,', 'organizers', 'teammates', 'kaggle', 'team', '@[', 'congratulations', 'winners', 'competition.']\ncustom_stopwords = my_custom_words + custom_stopwords\n#https://stackoverflow.com/questions/19560498/faster-way-to-remove-stop-words-in-python\nstopwords_dict = Counter(custom_stopwords)\n\n#custom_stopwords[:15]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:31:54.909629Z","iopub.execute_input":"2023-06-10T12:31:54.910195Z","iopub.status.idle":"2023-06-10T12:31:54.927789Z","shell.execute_reply.started":"2023-06-10T12:31:54.910159Z","shell.execute_reply":"2023-06-10T12:31:54.926412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_writeup_stopwords = ' '.join([word for word in str(clean_writeup_tokens).lower().split() if word not in stopwords_dict])\nclean_writeup_stopwords = ' '.join( [w for w in clean_writeup_stopwords.split() if len(w)>1] )","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:31:54.930526Z","iopub.execute_input":"2023-06-10T12:31:54.931382Z","iopub.status.idle":"2023-06-10T12:31:56.295443Z","shell.execute_reply.started":"2023-06-10T12:31:54.931336Z","shell.execute_reply":"2023-06-10T12:31:56.294420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_fd, bigram_fd, trigram_fd = freqDist_ngrams(clean_writeup_stopwords)\n\nbigram_fd.most_common()[40:80]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:31:56.297244Z","iopub.execute_input":"2023-06-10T12:31:56.298043Z","iopub.status.idle":"2023-06-10T12:32:01.449253Z","shell.execute_reply.started":"2023-06-10T12:31:56.298000Z","shell.execute_reply":"2023-06-10T12:32:01.447697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_cloud_generating(clean_writeup_stopwords, \"Clean and stopwords\", set(custom_stopwords))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:01.452900Z","iopub.execute_input":"2023-06-10T12:32:01.453375Z","iopub.status.idle":"2023-06-10T12:32:20.027621Z","shell.execute_reply.started":"2023-06-10T12:32:01.453337Z","shell.execute_reply":"2023-06-10T12:32:20.026399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cut the longer texts off and only use the first 512 Tokens\nBERT has a max length limit of tokens = 512. So this text has a length of much bigger than 512.\n\nWe have basically three options:\n\n- You can cut the longer texts off and only use the first 512 Tokens. The original BERT implementation (and probably the others as well) truncates longer sequences automatically. For most cases, this option is sufficient.\n- You can split your text in multiple subtexts, classify each of them and combine the results back together ( choose the class which was predicted for most of the subtexts for example). This option is obviously more expensive.\n- You can even feed the output token for each subtext (as in option 2) to another network (but you won't be able to fine-tune) as described in [this discussion](https://github.com/google-research/bert/issues/27).\n\n[Source](http://stackoverflow.com/questions/58636587/how-to-use-bert-for-long-text-classification)","metadata":{}},{"cell_type":"code","source":"\ndef cut_longer_text(row):\n    row = re.sub(r'[^\\w\\s]',' ',row)\n    clean_row = ' '.join([word for word in str(row).lower().split() if word not in stopwords_dict])\n    clean_row = ' '.join( [w for w in clean_row.split() if len(w)>1] )\n    return clean_row\n\ntrain_kw_pr['cut_longer_text'] = train_kw_ex['clean_writeup'].apply(cut_longer_text)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:20.029288Z","iopub.execute_input":"2023-06-10T12:32:20.029675Z","iopub.status.idle":"2023-06-10T12:32:21.392990Z","shell.execute_reply.started":"2023-06-10T12:32:20.029644Z","shell.execute_reply":"2023-06-10T12:32:21.392157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_ex['num_words'].nlargest(n=5)\ntrain_kw_ex[train_kw_ex.num_words<513].shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:21.394186Z","iopub.execute_input":"2023-06-10T12:32:21.395394Z","iopub.status.idle":"2023-06-10T12:32:21.406919Z","shell.execute_reply.started":"2023-06-10T12:32:21.395358Z","shell.execute_reply":"2023-06-10T12:32:21.405473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_kw_pr['num_words'] = train_kw_pr['cut_longer_text'].str.split().apply(len)\ntrain_kw_pr['num_words'].nlargest(n=50)\ntrain_kw_pr[train_kw_pr.num_words<513].shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:21.408745Z","iopub.execute_input":"2023-06-10T12:32:21.409129Z","iopub.status.idle":"2023-06-10T12:32:21.555858Z","shell.execute_reply.started":"2023-06-10T12:32:21.409100Z","shell.execute_reply":"2023-06-10T12:32:21.554479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature selection\n### Drop the attributes that provide no useful information for the task.\n#### Clean ngrams which no information","metadata":{}},{"cell_type":"code","source":"list_trigram_joined = []\nlist_trigram_not_joined = []\nfor item in list_trigram_fd:\n    list_trigram_joined.append('_'.join(item))\n    list_trigram_not_joined.append(' '.join(item))\n\nlist_bigram_joined = []\nlist_bigram_not_joined = []\nfor item in list_bigram_fd:\n    list_bigram_joined.append('_'.join(item))\n    list_bigram_not_joined.append(' '.join(item))\n    \ndisplay(list_bigram_joined[:5])\ndisplay(list_bigram_not_joined[:5])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:21.558014Z","iopub.execute_input":"2023-06-10T12:32:21.558995Z","iopub.status.idle":"2023-06-10T12:32:23.491511Z","shell.execute_reply.started":"2023-06-10T12:32:21.558947Z","shell.execute_reply":"2023-06-10T12:32:23.490225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature engineering.\n### Decompose features (e.g., categorical with one-hot)\nUse Title of Competition as label","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder\n\nordinal_encoder = OrdinalEncoder(dtype=np.int32)\ntrain_kw_pr['label'] = ordinal_encoder.fit_transform(train_kw_pr['Title of Competition'].values.reshape(-1, 1))","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:23.496473Z","iopub.execute_input":"2023-06-10T12:32:23.496898Z","iopub.status.idle":"2023-06-10T12:32:23.507926Z","shell.execute_reply.started":"2023-06-10T12:32:23.496843Z","shell.execute_reply":"2023-06-10T12:32:23.506756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add promising transformations\nMuch of the code with transformers has been obtained from Chapter 2 of the book Natural Language Processing with Transformers: Building Language Applications with Hugging Face 1st Edition from O'Reilly by Lewis Tunstall (Author), Leandro von Werra (Author), Thomas Wolf (Author ) from ISBN: 9781098136796 \n#### Build a Classifier text model to *Writeup* with Hugging Face \n##### Convert Pandas DataFrame to a DatasetDict object","metadata":{}},{"cell_type":"code","source":"temp = train_kw_pr[['cut_longer_text', 'num_words', 'label']]\ntemp['id'] = train_kw_pr.index","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:23.509141Z","iopub.execute_input":"2023-06-10T12:32:23.510270Z","iopub.status.idle":"2023-06-10T12:32:23.527596Z","shell.execute_reply.started":"2023-06-10T12:32:23.510236Z","shell.execute_reply":"2023-06-10T12:32:23.526044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sample of dataset","metadata":{}},{"cell_type":"code","source":"temp = temp[temp.num_words< 513].reset_index().drop(columns=['index'])\n","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:23.529475Z","iopub.execute_input":"2023-06-10T12:32:23.529845Z","iopub.status.idle":"2023-06-10T12:32:23.546755Z","shell.execute_reply.started":"2023-06-10T12:32:23.529815Z","shell.execute_reply":"2023-06-10T12:32:23.545192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try to ensure that these samples are representative of the whole dataset.","metadata":{"execution":{"iopub.status.busy":"2023-06-05T17:33:36.750939Z","iopub.execute_input":"2023-06-05T17:33:36.751825Z","iopub.status.idle":"2023-06-05T17:33:36.760990Z","shell.execute_reply.started":"2023-06-05T17:33:36.751783Z","shell.execute_reply":"2023-06-05T17:33:36.759948Z"}}},{"cell_type":"code","source":"temp[\"num_words\"].describe(percentiles=[.20, .40, .60, .80]).T","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:32:23.548341Z","iopub.execute_input":"2023-06-10T12:32:23.548685Z","iopub.status.idle":"2023-06-10T12:32:23.568590Z","shell.execute_reply.started":"2023-06-10T12:32:23.548656Z","shell.execute_reply":"2023-06-10T12:32:23.567390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp[\"num_words_cat\"] = pd.cut(temp[\"num_words\"],\n                               bins=[0, 92, 166, 242, 337, 512],\n                               labels=[1, 2, 3, 4, 5])\n\nfrom sklearn.model_selection import StratifiedShuffleSplit\nsplit = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\nfor train_index, test_index in split.split(temp, temp[\"num_words_cat\"]):\n    strat_train_set = temp.loc[train_index]\n    strat_test_set = temp.loc[test_index]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:08.658359Z","iopub.execute_input":"2023-06-10T12:38:08.659812Z","iopub.status.idle":"2023-06-10T12:38:08.674332Z","shell.execute_reply.started":"2023-06-10T12:38:08.659770Z","shell.execute_reply":"2023-06-10T12:38:08.673129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set = strat_train_set.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:08.708913Z","iopub.execute_input":"2023-06-10T12:38:08.709620Z","iopub.status.idle":"2023-06-10T12:38:08.715790Z","shell.execute_reply.started":"2023-06-10T12:38:08.709564Z","shell.execute_reply":"2023-06-10T12:38:08.715010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set['num_words_cat'].hist(bins=20)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:08.760480Z","iopub.execute_input":"2023-06-10T12:38:08.761106Z","iopub.status.idle":"2023-06-10T12:38:09.147070Z","shell.execute_reply.started":"2023-06-10T12:38:08.761073Z","shell.execute_reply":"2023-06-10T12:38:09.146149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = strat_test_set.copy()\ntemp = temp.reset_index()\n\nsplit = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=42)\nfor val_index, test_index in split.split(temp, temp[\"num_words_cat\"]):\n    strat_val_set = temp.loc[val_index]\n    strat_test_set = temp.loc[test_index]","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.148806Z","iopub.execute_input":"2023-06-10T12:38:09.149364Z","iopub.status.idle":"2023-06-10T12:38:09.159821Z","shell.execute_reply.started":"2023-06-10T12:38:09.149333Z","shell.execute_reply":"2023-06-10T12:38:09.158637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set = strat_train_set.drop(columns=['index', 'num_words_cat' ])\nstrat_val_set = strat_val_set.drop(columns=['index', 'num_words_cat' ])\nstrat_test_set = strat_test_set.drop(columns=['index', 'num_words_cat'])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.161251Z","iopub.execute_input":"2023-06-10T12:38:09.162123Z","iopub.status.idle":"2023-06-10T12:38:09.177959Z","shell.execute_reply.started":"2023-06-10T12:38:09.162086Z","shell.execute_reply":"2023-06-10T12:38:09.176498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set.shape, strat_val_set.shape, strat_test_set.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.181013Z","iopub.execute_input":"2023-06-10T12:38:09.182226Z","iopub.status.idle":"2023-06-10T12:38:09.193557Z","shell.execute_reply.started":"2023-06-10T12:38:09.182157Z","shell.execute_reply":"2023-06-10T12:38:09.192589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.194681Z","iopub.execute_input":"2023-06-10T12:38:09.195715Z","iopub.status.idle":"2023-06-10T12:38:09.216503Z","shell.execute_reply.started":"2023-06-10T12:38:09.195680Z","shell.execute_reply":"2023-06-10T12:38:09.215065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset, DatasetDict\nkw_dataset_tr = Dataset.from_pandas(strat_train_set)\nkw_dataset_va = Dataset.from_pandas(strat_val_set)\nkw_dataset_te = Dataset.from_pandas(strat_test_set)\n\n# https://stackoverflow.com/questions/67852880/how-can-i-handle-this-datasets-to-create-a-datasetdict\nkw_dataset = DatasetDict({\"train\":kw_dataset_tr,\"validation\":kw_dataset_va, \"test\":kw_dataset_te})","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.218933Z","iopub.execute_input":"2023-06-10T12:38:09.219366Z","iopub.status.idle":"2023-06-10T12:38:09.245086Z","shell.execute_reply.started":"2023-06-10T12:38:09.219318Z","shell.execute_reply":"2023-06-10T12:38:09.243826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Tokenizing the Whole Dataset","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\nmodel_ckpt = \"distilbert-base-uncased\"\ntokenizer = AutoTokenizer.from_pretrained(model_ckpt)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.247043Z","iopub.execute_input":"2023-06-10T12:38:09.247422Z","iopub.status.idle":"2023-06-10T12:38:09.374062Z","shell.execute_reply.started":"2023-06-10T12:38:09.247390Z","shell.execute_reply":"2023-06-10T12:38:09.372967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(batch):\n    return tokenizer(batch[\"cut_longer_text\"], padding=True, truncation=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.375445Z","iopub.execute_input":"2023-06-10T12:38:09.375754Z","iopub.status.idle":"2023-06-10T12:38:09.382012Z","shell.execute_reply.started":"2023-06-10T12:38:09.375728Z","shell.execute_reply":"2023-06-10T12:38:09.380558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kw_encoded = kw_dataset.map(tokenize, batched=True, batch_size=None)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.385366Z","iopub.execute_input":"2023-06-10T12:38:09.386199Z","iopub.status.idle":"2023-06-10T12:38:09.865976Z","shell.execute_reply.started":"2023-06-10T12:38:09.386165Z","shell.execute_reply":"2023-06-10T12:38:09.864836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check some Subword Tokenization","metadata":{}},{"cell_type":"code","source":"kw_encoded['train'].column_names","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:09.867475Z","iopub.execute_input":"2023-06-10T12:38:09.868070Z","iopub.status.idle":"2023-06-10T12:38:09.876010Z","shell.execute_reply.started":"2023-06-10T12:38:09.868037Z","shell.execute_reply":"2023-06-10T12:38:09.874657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set.loc[284]['cut_longer_text']","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:23.968860Z","iopub.execute_input":"2023-06-10T12:38:23.969361Z","iopub.status.idle":"2023-06-10T12:38:23.978353Z","shell.execute_reply.started":"2023-06-10T12:38:23.969327Z","shell.execute_reply":"2023-06-10T12:38:23.977162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strat_train_set['num_words'].sort_values(ascending=False).head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:24.028173Z","iopub.execute_input":"2023-06-10T12:38:24.029340Z","iopub.status.idle":"2023-06-10T12:38:24.039088Z","shell.execute_reply.started":"2023-06-10T12:38:24.029295Z","shell.execute_reply":"2023-06-10T12:38:24.037718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Training a Text Classifier with Transformers as Feature Extractors\nSample of dataset","metadata":{}},{"cell_type":"code","source":"from transformers import AutoModel\nimport torch\n\nmodel_ckpt = \"distilbert-base-uncased\"\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = AutoModel.from_pretrained(model_ckpt).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:24.065375Z","iopub.execute_input":"2023-06-10T12:38:24.065830Z","iopub.status.idle":"2023-06-10T12:38:39.379724Z","shell.execute_reply.started":"2023-06-10T12:38:24.065791Z","shell.execute_reply":"2023-06-10T12:38:39.378479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_hidden_states(batch):\n    # Place model inputs on the GPU\n    inputs = {k:v.to(device) for k,v in batch.items() \n              if k in tokenizer.model_input_names}\n    # Extract last hidden states\n    with torch.no_grad():\n        last_hidden_state = model(**inputs).last_hidden_state\n    # Return vector for [CLS] token\n    return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:39.381969Z","iopub.execute_input":"2023-06-10T12:38:39.382343Z","iopub.status.idle":"2023-06-10T12:38:39.390594Z","shell.execute_reply.started":"2023-06-10T12:38:39.382312Z","shell.execute_reply":"2023-06-10T12:38:39.389090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" kw_encoded.set_format(\"torch\",\n                       columns=[\"input_ids\", \"attention_mask\", \"label\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:39.392743Z","iopub.execute_input":"2023-06-10T12:38:39.393229Z","iopub.status.idle":"2023-06-10T12:38:39.414981Z","shell.execute_reply.started":"2023-06-10T12:38:39.393185Z","shell.execute_reply":"2023-06-10T12:38:39.413872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/71498324/pytorch-runtimeerror-cuda-out-of-memory-with-a-huge-amount-of-free-memory\nkw_hidden = kw_encoded.map(extract_hidden_states, batched=True, batch_size=20)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:38:39.418414Z","iopub.execute_input":"2023-06-10T12:38:39.418820Z","iopub.status.idle":"2023-06-10T12:44:08.761545Z","shell.execute_reply.started":"2023-06-10T12:38:39.418792Z","shell.execute_reply":"2023-06-10T12:44:08.760048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Creating a feature matrix","metadata":{}},{"cell_type":"code","source":"X_train = np.array(kw_hidden[\"train\"][\"hidden_state\"])\nX_valid = np.array(kw_hidden[\"validation\"][\"hidden_state\"])\ny_train = np.array(kw_hidden[\"train\"][\"label\"])\ny_valid = np.array(kw_hidden[\"validation\"][\"label\"])\nX_train.shape, X_valid.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:44:08.763421Z","iopub.execute_input":"2023-06-10T12:44:08.764023Z","iopub.status.idle":"2023-06-10T12:44:08.783049Z","shell.execute_reply.started":"2023-06-10T12:44:08.763989Z","shell.execute_reply":"2023-06-10T12:44:08.780970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from umap import UMAP\nfrom sklearn.preprocessing import MinMaxScaler\n\n# Scale features to [0,1] range\nX_scaled = MinMaxScaler().fit_transform(X_train)\n# Initialize and fit UMAP\nmapper = UMAP(n_components=2, metric=\"cosine\").fit(X_scaled)\n# Create a DataFrame of 2D embeddings\ndf_emb = pd.DataFrame(mapper.embedding_, columns=[\"X\", \"Y\"])\ndf_emb[\"label\"] = y_train\ndf_emb.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:44:08.784939Z","iopub.execute_input":"2023-06-10T12:44:08.785785Z","iopub.status.idle":"2023-06-10T12:44:42.854407Z","shell.execute_reply.started":"2023-06-10T12:44:08.785750Z","shell.execute_reply":"2023-06-10T12:44:42.852955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Training a simple classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlr_clf = LogisticRegression(max_iter=3000)\nlr_clf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:44:42.856135Z","iopub.execute_input":"2023-06-10T12:44:42.856477Z","iopub.status.idle":"2023-06-10T12:45:07.186316Z","shell.execute_reply.started":"2023-06-10T12:44:42.856450Z","shell.execute_reply":"2023-06-10T12:45:07.184441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_clf.score(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:45:07.193977Z","iopub.execute_input":"2023-06-10T12:45:07.199615Z","iopub.status.idle":"2023-06-10T12:45:07.227034Z","shell.execute_reply.started":"2023-06-10T12:45:07.199544Z","shell.execute_reply":"2023-06-10T12:45:07.225398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.dummy import DummyClassifier\n\ndummy_clf = DummyClassifier(strategy=\"most_frequent\")\ndummy_clf.fit(X_train, y_train)\ndummy_clf.score(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:45:07.234416Z","iopub.execute_input":"2023-06-10T12:45:07.239943Z","iopub.status.idle":"2023-06-10T12:45:07.272371Z","shell.execute_reply.started":"2023-06-10T12:45:07.239860Z","shell.execute_reply":"2023-06-10T12:45:07.270898Z"},"trusted":true},"execution_count":null,"outputs":[]}]}