{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing all the required libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.snowball import SnowballStemmer\n\nimport nltk\nnltk.download('wordnet')\n! unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/\nfrom nltk.stem import WordNetLemmatizer\n\nimport re\nfrom wordcloud import WordCloud\nfrom bs4 import BeautifulSoup\n\nfrom sklearn.model_selection import cross_val_score\n\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score,precision_score,recall_score,accuracy_score,classification_report","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:08:44.627492Z","iopub.execute_input":"2023-07-14T17:08:44.627928Z","iopub.status.idle":"2023-07-14T17:08:47.594060Z","shell.execute_reply.started":"2023-07-14T17:08:44.627895Z","shell.execute_reply":"2023-07-14T17:08:47.593108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip /kaggle/input/facebook-recruiting-iii-keyword-extraction/Train.zip","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:30:17.294114Z","iopub.execute_input":"2023-07-14T17:30:17.294465Z","iopub.status.idle":"2023-07-14T17:31:35.617846Z","shell.execute_reply.started":"2023-07-14T17:30:17.294440Z","shell.execute_reply":"2023-07-14T17:31:35.614669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/working/Train.csv').sample(500000)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:31:46.512380Z","iopub.execute_input":"2023-07-14T17:31:46.513040Z","iopub.status.idle":"2023-07-14T17:33:44.353773Z","shell.execute_reply.started":"2023-07-14T17:31:46.512989Z","shell.execute_reply":"2023-07-14T17:33:44.352006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the data","metadata":{}},{"cell_type":"code","source":"print(f'The shape of the Train dataset: {df_train.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:12.852645Z","iopub.execute_input":"2023-07-14T17:34:12.853181Z","iopub.status.idle":"2023-07-14T17:34:13.213748Z","shell.execute_reply.started":"2023-07-14T17:34:12.853153Z","shell.execute_reply":"2023-07-14T17:34:13.211986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.reset_index(drop = True,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:16.639754Z","iopub.execute_input":"2023-07-14T17:34:16.640255Z","iopub.status.idle":"2023-07-14T17:34:17.291543Z","shell.execute_reply.started":"2023-07-14T17:34:16.640229Z","shell.execute_reply":"2023-07-14T17:34:17.289044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:30.499010Z","iopub.execute_input":"2023-07-14T17:34:30.499584Z","iopub.status.idle":"2023-07-14T17:34:33.040907Z","shell.execute_reply.started":"2023-07-14T17:34:30.499546Z","shell.execute_reply":"2023-07-14T17:34:33.039624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Body'][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:38.372796Z","iopub.execute_input":"2023-07-14T17:34:38.373267Z","iopub.status.idle":"2023-07-14T17:34:38.383807Z","shell.execute_reply.started":"2023-07-14T17:34:38.373230Z","shell.execute_reply":"2023-07-14T17:34:38.382149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Tags'][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:41.413651Z","iopub.execute_input":"2023-07-14T17:34:41.414106Z","iopub.status.idle":"2023-07-14T17:34:41.423319Z","shell.execute_reply.started":"2023-07-14T17:34:41.414055Z","shell.execute_reply":"2023-07-14T17:34:41.422162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train DataFrame Shape: {df_train.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:43.208269Z","iopub.execute_input":"2023-07-14T17:34:43.208671Z","iopub.status.idle":"2023-07-14T17:34:43.216263Z","shell.execute_reply.started":"2023-07-14T17:34:43.208644Z","shell.execute_reply":"2023-07-14T17:34:43.214300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing","metadata":{}},{"cell_type":"markdown","source":"### Checking for duplicates","metadata":{}},{"cell_type":"code","source":"duplicated_pairs = df_train.sort_values('Title', ascending=False).duplicated('Title')\nprint(f'Total number of duplicated questions: {duplicated_pairs.sum()}')\n\ndf_train = df_train[~duplicated_pairs]\nprint(f'Dataframe shape after duplicate removal: {df_train.shape}')\n","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:34:46.751683Z","iopub.execute_input":"2023-07-14T17:34:46.752155Z","iopub.status.idle":"2023-07-14T17:34:49.759601Z","shell.execute_reply.started":"2023-07-14T17:34:46.752107Z","shell.execute_reply":"2023-07-14T17:34:49.757768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:36:49.752048Z","iopub.execute_input":"2023-07-14T17:36:49.752537Z","iopub.status.idle":"2023-07-14T17:36:50.672916Z","shell.execute_reply.started":"2023-07-14T17:36:49.752501Z","shell.execute_reply":"2023-07-14T17:36:50.671247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:36:51.667579Z","iopub.execute_input":"2023-07-14T17:36:51.668877Z","iopub.status.idle":"2023-07-14T17:36:52.451474Z","shell.execute_reply.started":"2023-07-14T17:36:51.668826Z","shell.execute_reply":"2023-07-14T17:36:52.449715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.dropna(axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:36:53.400048Z","iopub.execute_input":"2023-07-14T17:36:53.400499Z","iopub.status.idle":"2023-07-14T17:36:54.282967Z","shell.execute_reply.started":"2023-07-14T17:36:53.400467Z","shell.execute_reply":"2023-07-14T17:36:54.281313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:36:55.219870Z","iopub.execute_input":"2023-07-14T17:36:55.220314Z","iopub.status.idle":"2023-07-14T17:36:55.958939Z","shell.execute_reply.started":"2023-07-14T17:36:55.220279Z","shell.execute_reply":"2023-07-14T17:36:55.956878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pprint import pprint\n\npprint(df_train.iloc[3].to_dict(), compact=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:36:58.680654Z","iopub.execute_input":"2023-07-14T17:36:58.681047Z","iopub.status.idle":"2023-07-14T17:36:58.689898Z","shell.execute_reply.started":"2023-07-14T17:36:58.681017Z","shell.execute_reply":"2023-07-14T17:36:58.688103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating a tag count column as a new feature","metadata":{}},{"cell_type":"code","source":"df_train['Tag_Counts'] = df_train['Tags'].apply(lambda x: len(x.split()))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:10.588198Z","iopub.execute_input":"2023-07-14T17:37:10.589546Z","iopub.status.idle":"2023-07-14T17:37:10.996284Z","shell.execute_reply.started":"2023-07-14T17:37:10.589494Z","shell.execute_reply":"2023-07-14T17:37:10.994904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[['Tags','Tag_Counts']]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:12.091344Z","iopub.execute_input":"2023-07-14T17:37:12.092197Z","iopub.status.idle":"2023-07-14T17:37:12.138464Z","shell.execute_reply.started":"2023-07-14T17:37:12.092127Z","shell.execute_reply":"2023-07-14T17:37:12.136106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic Data Analysis on Tags","metadata":{}},{"cell_type":"markdown","source":"### Frequency of tag_count","metadata":{}},{"cell_type":"code","source":"df_train['Tag_Counts'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:14.269469Z","iopub.execute_input":"2023-07-14T17:37:14.270279Z","iopub.status.idle":"2023-07-14T17:37:14.291207Z","shell.execute_reply.started":"2023-07-14T17:37:14.270231Z","shell.execute_reply":"2023-07-14T17:37:14.289473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( \"Maximum number of tags in a question: \", df_train[\"Tag_Counts\"].max())\nprint( \"Minimum number of tags in a question: \", df_train[\"Tag_Counts\"].min())\nprint( \"Average number of tags in a question: \", df_train[\"Tag_Counts\"].mean())","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:17.796719Z","iopub.execute_input":"2023-07-14T17:37:17.797109Z","iopub.status.idle":"2023-07-14T17:37:17.807685Z","shell.execute_reply.started":"2023-07-14T17:37:17.797067Z","shell.execute_reply":"2023-07-14T17:37:17.805803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:21.147331Z","iopub.execute_input":"2023-07-14T17:37:21.147823Z","iopub.status.idle":"2023-07-14T17:37:21.173447Z","shell.execute_reply.started":"2023-07-14T17:37:21.147796Z","shell.execute_reply":"2023-07-14T17:37:21.170462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Something looks odd about this","metadata":{"execution":{"iopub.status.busy":"2023-07-06T16:11:45.413109Z","iopub.execute_input":"2023-07-06T16:11:45.413491Z","iopub.status.idle":"2023-07-06T16:11:45.418499Z","shell.execute_reply.started":"2023-07-06T16:11:45.413461Z","shell.execute_reply":"2023-07-06T16:11:45.417148Z"}}},{"cell_type":"code","source":"df_train['Tag_Counts'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:23.987374Z","iopub.execute_input":"2023-07-14T17:37:23.987832Z","iopub.status.idle":"2023-07-14T17:37:24.006499Z","shell.execute_reply.started":"2023-07-14T17:37:23.987798Z","shell.execute_reply":"2023-07-14T17:37:24.003894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_tag_freq = df_train['Tag_Counts'].value_counts().reset_index().sort_values(by='index').rename(columns = {'index':'tag_count', 'Tag_Counts':'tag_value_counts'})\n\n\nplt.title('Number of tags in questions')\nsns.barplot(data = df_train_tag_freq, x = 'tag_count', y = 'tag_value_counts')\nplt.xlabel('Number of Tags')\nplt.ylabel('Frquency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:26.926214Z","iopub.execute_input":"2023-07-14T17:37:26.926641Z","iopub.status.idle":"2023-07-14T17:37:27.238762Z","shell.execute_reply.started":"2023-07-14T17:37:26.926607Z","shell.execute_reply":"2023-07-14T17:37:27.236654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Observations: \n1. Maximum number of tags in question: 5\n2. Minimum number of tags in question: 1\n3. Average number of tags in question: 2.89\n4. Most of the questions have either 2 or 3 tags to work with.","metadata":{}},{"cell_type":"markdown","source":"### Total number of unique Tags","metadata":{}},{"cell_type":"code","source":"vectorizer = CountVectorizer(tokenizer = lambda x : x.split())\ntag_bow = vectorizer.fit_transform(df_train['Tags'])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:31.347475Z","iopub.execute_input":"2023-07-14T17:37:31.347950Z","iopub.status.idle":"2023-07-14T17:37:34.021646Z","shell.execute_reply.started":"2023-07-14T17:37:31.347899Z","shell.execute_reply":"2023-07-14T17:37:34.020110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_bow.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:36.459538Z","iopub.execute_input":"2023-07-14T17:37:36.459904Z","iopub.status.idle":"2023-07-14T17:37:36.466503Z","shell.execute_reply.started":"2023-07-14T17:37:36.459881Z","shell.execute_reply":"2023-07-14T17:37:36.464707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of questions :\", tag_bow.shape[0])\nprint(\"Number of unique tags :\", tag_bow.shape[1])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:37.723751Z","iopub.execute_input":"2023-07-14T17:37:37.724215Z","iopub.status.idle":"2023-07-14T17:37:37.730100Z","shell.execute_reply.started":"2023-07-14T17:37:37.724179Z","shell.execute_reply":"2023-07-14T17:37:37.729373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tags = vectorizer.get_feature_names_out()\nprint(\"Few tags :\", tags[:10])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:40.857181Z","iopub.execute_input":"2023-07-14T17:37:40.858076Z","iopub.status.idle":"2023-07-14T17:37:40.906483Z","shell.execute_reply.started":"2023-07-14T17:37:40.858039Z","shell.execute_reply":"2023-07-14T17:37:40.904185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:43.273896Z","iopub.execute_input":"2023-07-14T17:37:43.274785Z","iopub.status.idle":"2023-07-14T17:37:43.289248Z","shell.execute_reply.started":"2023-07-14T17:37:43.274726Z","shell.execute_reply":"2023-07-14T17:37:43.287062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Frequency of each tag","metadata":{}},{"cell_type":"code","source":"freq = tag_bow.sum(axis = 0).A1\ntag_to_count_map = dict(zip(tags,freq))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:45.984700Z","iopub.execute_input":"2023-07-14T17:37:45.985164Z","iopub.status.idle":"2023-07-14T17:37:46.015142Z","shell.execute_reply.started":"2023-07-14T17:37:45.985132Z","shell.execute_reply":"2023-07-14T17:37:46.012329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"freq","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:52.048905Z","iopub.execute_input":"2023-07-14T17:37:52.049398Z","iopub.status.idle":"2023-07-14T17:37:52.060529Z","shell.execute_reply.started":"2023-07-14T17:37:52.049362Z","shell.execute_reply":"2023-07-14T17:37:52.058831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Shape of Freqeuency: {freq.shape}')\nprint(f'Shape of Tags: {tags.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:52.869888Z","iopub.execute_input":"2023-07-14T17:37:52.871330Z","iopub.status.idle":"2023-07-14T17:37:52.879331Z","shell.execute_reply.started":"2023-07-14T17:37:52.871266Z","shell.execute_reply":"2023-07-14T17:37:52.877870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list = []\nfor key, value in tag_to_count_map.items():\n list.append([key, value]) \n\n\ntag_df_train = pd.DataFrame(list, columns=['Tags', 'Counts'])\ntag_df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:53.652130Z","iopub.execute_input":"2023-07-14T17:37:53.652604Z","iopub.status.idle":"2023-07-14T17:37:53.780903Z","shell.execute_reply.started":"2023-07-14T17:37:53.652569Z","shell.execute_reply":"2023-07-14T17:37:53.779192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_df_train_sorted = tag_df_train.sort_values(['Counts'], ascending=False)\nplt.plot(tag_df_train_sorted['Counts'].values)\nplt.grid()\nplt.title(\"Distribution of frequency of tags based on appeareance\")\nplt.xlabel(\"Tag numbers for most frequent tags\")\nplt.ylabel(\"Frequency\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:37:56.496875Z","iopub.execute_input":"2023-07-14T17:37:56.497760Z","iopub.status.idle":"2023-07-14T17:37:56.801247Z","shell.execute_reply.started":"2023-07-14T17:37:56.497714Z","shell.execute_reply":"2023-07-14T17:37:56.799903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(tag_df_train_sorted['Counts'][0:100].values)\nplt.grid()\nplt.title(\"Top 100 tags : Distribution of frequency of tags based on appeareance\")\nplt.xlabel(\"Tag numbers for most frequent tags\")\nplt.ylabel(\"Frequency\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:00.749740Z","iopub.execute_input":"2023-07-14T17:38:00.750198Z","iopub.status.idle":"2023-07-14T17:38:01.012794Z","shell.execute_reply.started":"2023-07-14T17:38:00.750159Z","shell.execute_reply":"2023-07-14T17:38:01.010555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_bow","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:05.025822Z","iopub.execute_input":"2023-07-14T17:38:05.026271Z","iopub.status.idle":"2023-07-14T17:38:05.036081Z","shell.execute_reply.started":"2023-07-14T17:38:05.026242Z","shell.execute_reply":"2023-07-14T17:38:05.034229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data pre-processing\n\n- Cleaning up the title and body\n- realtion with name of the tag and title or body\n- Present of code snippet based on body. \n- Length of the body\n- if any attachment with the questions. \n- Url attachments\n- Synonymn of tags in title or body.\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T16:59:15.360998Z","iopub.execute_input":"2023-07-06T16:59:15.361442Z","iopub.status.idle":"2023-07-06T16:59:15.372534Z","shell.execute_reply.started":"2023-07-06T16:59:15.361410Z","shell.execute_reply":"2023-07-06T16:59:15.369876Z"}}},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:06.483220Z","iopub.execute_input":"2023-07-14T17:38:06.483714Z","iopub.status.idle":"2023-07-14T17:38:06.502621Z","shell.execute_reply.started":"2023-07-14T17:38:06.483680Z","shell.execute_reply":"2023-07-14T17:38:06.500613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Body'][1]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:09.608598Z","iopub.execute_input":"2023-07-14T17:38:09.609080Z","iopub.status.idle":"2023-07-14T17:38:09.630423Z","shell.execute_reply.started":"2023-07-14T17:38:09.609046Z","shell.execute_reply":"2023-07-14T17:38:09.628657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_df_train_sorted","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:14.339151Z","iopub.execute_input":"2023-07-14T17:38:14.339566Z","iopub.status.idle":"2023-07-14T17:38:14.356734Z","shell.execute_reply.started":"2023-07-14T17:38:14.339534Z","shell.execute_reply":"2023-07-14T17:38:14.354623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"{} tags are used more than 25 times\".format(tag_df_train_sorted[tag_df_train_sorted[\"Counts\"]>25].shape[0]))\nprint(\"{} tags are used more than 50 times\".format(tag_df_train_sorted[tag_df_train_sorted[\"Counts\"]>50].shape[0]))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:17.844196Z","iopub.execute_input":"2023-07-14T17:38:17.844563Z","iopub.status.idle":"2023-07-14T17:38:17.856423Z","shell.execute_reply.started":"2023-07-14T17:38:17.844540Z","shell.execute_reply":"2023-07-14T17:38:17.853780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations:\n1. 144 tags are used more than 25 times.\n2. 59 tags are used more than 50 times.\n3. C# is most frequently used tag 778 times.\n4. Since some tags occur much more frequenctly than others, Micro-averaged F1-score is the appropriate metric for this probelm.","metadata":{}},{"cell_type":"markdown","source":"## Word map","metadata":{}},{"cell_type":"code","source":"tag_to_count_map\ntupl = dict(tag_to_count_map.items())\nword_cloud = WordCloud(width=1600,height=800,).generate_from_frequencies(tupl)\nplt.figure(figsize = (12,8))\nplt.imshow(word_cloud)\nplt.axis('off')\nplt.tight_layout(pad=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:38:21.237405Z","iopub.execute_input":"2023-07-14T17:38:21.237782Z","iopub.status.idle":"2023-07-14T17:38:24.810097Z","shell.execute_reply.started":"2023-07-14T17:38:21.237757Z","shell.execute_reply":"2023-07-14T17:38:24.809161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Barplot of top 20 tags","metadata":{}},{"cell_type":"code","source":"tag_df_train_sorted.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:02.886589Z","iopub.execute_input":"2023-07-14T17:43:02.887009Z","iopub.status.idle":"2023-07-14T17:43:02.902861Z","shell.execute_reply.started":"2023-07-14T17:43:02.886980Z","shell.execute_reply":"2023-07-14T17:43:02.901501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=np.arange(25)\ntag_df_train_sorted.head(25).plot(kind='bar')\nplt.title('Frequency of top 25 tags')\nplt.xticks(i, tag_df_train_sorted['Tags'].head(25))\nplt.xlabel('Tags')\nplt.ylabel('Counts')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:03.389996Z","iopub.execute_input":"2023-07-14T17:43:03.390464Z","iopub.status.idle":"2023-07-14T17:43:03.919583Z","shell.execute_reply.started":"2023-07-14T17:43:03.390430Z","shell.execute_reply":"2023-07-14T17:43:03.917723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"{} tags are used more than 25 times\".format(tag_df_train_sorted[tag_df_train_sorted[\"Counts\"]>25].shape[0]))\nprint(\"{} tags are used more than 50 times\".format(tag_df_train_sorted[tag_df_train_sorted[\"Counts\"]>50].shape[0]))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:04.816336Z","iopub.execute_input":"2023-07-14T17:43:04.816765Z","iopub.status.idle":"2023-07-14T17:43:04.830289Z","shell.execute_reply.started":"2023-07-14T17:43:04.816732Z","shell.execute_reply":"2023-07-14T17:43:04.828420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_df_train_sorted['Tags']","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:05.638447Z","iopub.execute_input":"2023-07-14T17:43:05.638779Z","iopub.status.idle":"2023-07-14T17:43:05.647234Z","shell.execute_reply.started":"2023-07-14T17:43:05.638757Z","shell.execute_reply":"2023-07-14T17:43:05.646075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Body_Length'] = df_train['Body'].apply(lambda x : len(x.split()))\nsns.scatterplot(x= df_train['Body_Length'], y = df_train['Tag_Counts'])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:08.121825Z","iopub.execute_input":"2023-07-14T17:43:08.122480Z","iopub.status.idle":"2023-07-14T17:43:14.829074Z","shell.execute_reply.started":"2023-07-14T17:43:08.122446Z","shell.execute_reply":"2023-07-14T17:43:14.827377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Title_Length'] = df_train['Title'].apply(lambda x : len(x.split()))\nsns.scatterplot(x= df_train['Title_Length'], y = df_train['Tag_Counts'])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:14.831549Z","iopub.execute_input":"2023-07-14T17:43:14.831895Z","iopub.status.idle":"2023-07-14T17:43:16.836302Z","shell.execute_reply.started":"2023-07-14T17:43:14.831864Z","shell.execute_reply":"2023-07-14T17:43:16.834648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Text preprocessing","metadata":{}},{"cell_type":"code","source":"def find_code(df):\n if '' in df:\n return 1\n else: \n return 0\n\n \ndf_train['is_code'] = df_train['Body'].apply(find_code)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:20.384970Z","iopub.execute_input":"2023-07-14T17:43:20.385427Z","iopub.status.idle":"2023-07-14T17:43:20.984107Z","shell.execute_reply.started":"2023-07-14T17:43:20.385395Z","shell.execute_reply":"2023-07-14T17:43:20.983201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:21.906315Z","iopub.execute_input":"2023-07-14T17:43:21.907525Z","iopub.status.idle":"2023-07-14T17:43:21.926139Z","shell.execute_reply.started":"2023-07-14T17:43:21.907455Z","shell.execute_reply":"2023-07-14T17:43:21.925066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"There are a total of {df_train[df_train['is_code'] == 1].shape[0]} questions with code out of {df_train.shape[0]}\")\nprint(f\"{np.round((100 * df_train[df_train['is_code'] == 1].shape[0])/df_train.shape[0],2)}% of the questions has code\")","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:28.492330Z","iopub.execute_input":"2023-07-14T17:43:28.492761Z","iopub.status.idle":"2023-07-14T17:43:28.688859Z","shell.execute_reply.started":"2023-07-14T17:43:28.492728Z","shell.execute_reply":"2023-07-14T17:43:28.687203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:31.926770Z","iopub.execute_input":"2023-07-14T17:43:31.927464Z","iopub.status.idle":"2023-07-14T17:43:31.945811Z","shell.execute_reply.started":"2023-07-14T17:43:31.927427Z","shell.execute_reply":"2023-07-14T17:43:31.943840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train['is_code'] == 1]['Body'].reset_index().iloc[4,:]['Body']","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:35.100361Z","iopub.execute_input":"2023-07-14T17:43:35.101785Z","iopub.status.idle":"2023-07-14T17:43:35.226990Z","shell.execute_reply.started":"2023-07-14T17:43:35.101724Z","shell.execute_reply":"2023-07-14T17:43:35.225765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pprint(df_train[df_train['is_code'] == 1].iloc[1].to_dict(), compact=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:39.048009Z","iopub.execute_input":"2023-07-14T17:43:39.048467Z","iopub.status.idle":"2023-07-14T17:43:39.148470Z","shell.execute_reply.started":"2023-07-14T17:43:39.048442Z","shell.execute_reply":"2023-07-14T17:43:39.146849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def striphtml(data):\n cleanr = re.compile('<.*?>')\n cleantext = re.sub(cleanr,' ',str(data))\n return cleantext\n\n\n\nstop_words = set(stopwords.words('english'))\nstemmer = SnowballStemmer('english')\nlemmatizer = WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:43.911480Z","iopub.execute_input":"2023-07-14T17:43:43.912181Z","iopub.status.idle":"2023-07-14T17:43:43.925493Z","shell.execute_reply.started":"2023-07-14T17:43:43.912140Z","shell.execute_reply":"2023-07-14T17:43:43.923829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 20 tags used in our corpus\nlist_tags = tag_df_train_sorted['Tags'].head(20).to_list()\n\n# Filtering by only those rows where the tags are in top 20\ndf_tags = df_train[(df_train['Tags'].isin(list_tags))][['Title','Body', 'Tags']]\n\ndf_tags.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:43:56.924347Z","iopub.execute_input":"2023-07-14T17:43:56.924773Z","iopub.status.idle":"2023-07-14T17:43:57.137697Z","shell.execute_reply.started":"2023-07-14T17:43:56.924744Z","shell.execute_reply":"2023-07-14T17:43:57.136621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:00.897350Z","iopub.execute_input":"2023-07-14T17:45:00.897811Z","iopub.status.idle":"2023-07-14T17:45:00.923506Z","shell.execute_reply.started":"2023-07-14T17:45:00.897776Z","shell.execute_reply":"2023-07-14T17:45:00.921028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note: Based on the currect logic ","metadata":{}},{"cell_type":"code","source":"df_train[(df_train['Tags'].isin(list_tags))][['Title','Body', 'Tags']]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:05.187071Z","iopub.execute_input":"2023-07-14T17:45:05.187517Z","iopub.status.idle":"2023-07-14T17:45:05.344072Z","shell.execute_reply.started":"2023-07-14T17:45:05.187487Z","shell.execute_reply":"2023-07-14T17:45:05.341963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tags.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:08.081263Z","iopub.execute_input":"2023-07-14T17:45:08.081684Z","iopub.status.idle":"2023-07-14T17:45:08.092134Z","shell.execute_reply.started":"2023-07-14T17:45:08.081652Z","shell.execute_reply":"2023-07-14T17:45:08.090240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_tags[df_tags['Title'] == 'Why is my cube only rendering as a square?']","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:17.943900Z","iopub.execute_input":"2023-07-14T17:45:17.944315Z","iopub.status.idle":"2023-07-14T17:45:17.950717Z","shell.execute_reply.started":"2023-07-14T17:45:17.944284Z","shell.execute_reply":"2023-07-14T17:45:17.948735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tags['Tags'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:20.522752Z","iopub.execute_input":"2023-07-14T17:45:20.523189Z","iopub.status.idle":"2023-07-14T17:45:20.537477Z","shell.execute_reply.started":"2023-07-14T17:45:20.523156Z","shell.execute_reply":"2023-07-14T17:45:20.535180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.now()\npreprocessed_data_list=[]\nquestions_with_code=0\nlen_pre=0\nlen_post=0\nquestions_proccesed = 0\nprepared_df = pd.DataFrame(columns=['question','code','tags','words_pre','words_post','is_code'])\n\n\nfor row in df_tags.iterrows():\n\n is_code = 0\n\n title, question, tags = row[1][0], row[1][1], row[1][2]\n\n if '' in question:\n questions_with_code+=1\n is_code = 1\n \n x = len(question)+len(title)\n len_pre+=x\n\n code = str(re.findall(r'(.*?)', question, flags=re.DOTALL))\n\n question=re.sub('(.*?)', '', question, flags=re.MULTILINE|re.DOTALL)\n question=striphtml(question.encode('utf-8'))\n\n title=title.encode('utf-8')\n\n question=str(title)+\" \"+str(question)\n question=re.sub(r'[^A-Za-z]+',' ',question)\n words=word_tokenize(str(question.lower()))\n\n #Removing all single letter and and stopwords from question exceptt for the letter 'c'\n question=' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))\n\n len_post+=len(question)\n processed_di = {\n \"question\": question,\n \"code\": code,\n \"tags\": tags,\n \"words_pre\": x,\n \"words_post\": len(question),\n \"is_code\" : is_code\n }\n \n prepared_df.loc[len(prepared_df.index)] = [question,code,tags,x,len(question),is_code]\n \n questions_proccesed += 1\n if (questions_proccesed%100000==0):\n print(\"number of questions completed=\",questions_proccesed)\n\nno_dup_avg_len_pre=(len_pre*1.0)/questions_proccesed\nno_dup_avg_len_post=(len_post*1.0)/questions_proccesed\n\nprint( \"Avg. length of questions(Title+Body) before processing: %d\"%no_dup_avg_len_pre)\nprint( \"Avg. length of questions(Title+Body) after processing: %d\"%no_dup_avg_len_post)\nprint (\"Percent of questions containing code: %d\"%((questions_with_code*100.0)/questions_proccesed))\n\nprint(\"Time taken to run this cell :\", datetime.now() - start)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:45:34.654451Z","iopub.execute_input":"2023-07-14T17:45:34.654865Z","iopub.status.idle":"2023-07-14T17:47:59.928895Z","shell.execute_reply.started":"2023-07-14T17:45:34.654829Z","shell.execute_reply":"2023-07-14T17:47:59.927059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prepared_df['tags'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:52:52.767435Z","iopub.execute_input":"2023-07-14T17:52:52.768364Z","iopub.status.idle":"2023-07-14T17:52:52.782954Z","shell.execute_reply.started":"2023-07-14T17:52:52.768290Z","shell.execute_reply":"2023-07-14T17:52:52.781251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pprint(df_tags.iloc[1].to_dict(), compact=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:52:56.588731Z","iopub.execute_input":"2023-07-14T17:52:56.589237Z","iopub.status.idle":"2023-07-14T17:52:56.597921Z","shell.execute_reply.started":"2023-07-14T17:52:56.589187Z","shell.execute_reply":"2023-07-14T17:52:56.596185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pprint(prepared_df.iloc[1].to_dict(), compact=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:52:59.672728Z","iopub.execute_input":"2023-07-14T17:52:59.673190Z","iopub.status.idle":"2023-07-14T17:52:59.683143Z","shell.execute_reply.started":"2023-07-14T17:52:59.673156Z","shell.execute_reply":"2023-07-14T17:52:59.680949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Original Df length: \\t' + str(len(df_train)))\nlen_df_tags = len(prepared_df)\nprint('df tags length: \\t' + str(len_df_tags))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:53:03.298668Z","iopub.execute_input":"2023-07-14T17:53:03.299101Z","iopub.status.idle":"2023-07-14T17:53:03.306503Z","shell.execute_reply.started":"2023-07-14T17:53:03.299053Z","shell.execute_reply":"2023-07-14T17:53:03.304890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prepared_df['question'][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:53:06.532502Z","iopub.execute_input":"2023-07-14T17:53:06.532947Z","iopub.status.idle":"2023-07-14T17:53:06.543595Z","shell.execute_reply.started":"2023-07-14T17:53:06.532915Z","shell.execute_reply":"2023-07-14T17:53:06.541763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prepared_df = prepared_df[['question','tags']]","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:53:09.457175Z","iopub.execute_input":"2023-07-14T17:53:09.457643Z","iopub.status.idle":"2023-07-14T17:53:09.479082Z","shell.execute_reply.started":"2023-07-14T17:53:09.457611Z","shell.execute_reply":"2023-07-14T17:53:09.477766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nfor items in prepared_df['question']:\n print(items)\n print('------------------------------------------------------------------------------')\n \n i += 1\n \n if i > 10:\n break","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:53:12.535944Z","iopub.execute_input":"2023-07-14T17:53:12.536896Z","iopub.status.idle":"2023-07-14T17:53:12.547353Z","shell.execute_reply.started":"2023-07-14T17:53:12.536859Z","shell.execute_reply":"2023-07-14T17:53:12.544977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ML Modeling","metadata":{}},{"cell_type":"markdown","source":"### Splitting the data for Train Test Split","metadata":{}},{"cell_type":"code","source":"X = prepared_df.question\ny = prepared_df.tags","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:54:21.114708Z","iopub.execute_input":"2023-07-14T17:54:21.115164Z","iopub.status.idle":"2023-07-14T17:54:21.120919Z","shell.execute_reply.started":"2023-07-14T17:54:21.115126Z","shell.execute_reply":"2023-07-14T17:54:21.119509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:54:33.172138Z","iopub.execute_input":"2023-07-14T17:54:33.172681Z","iopub.status.idle":"2023-07-14T17:54:33.185678Z","shell.execute_reply.started":"2023-07-14T17:54:33.172646Z","shell.execute_reply":"2023-07-14T17:54:33.183237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T17:54:38.121770Z","iopub.execute_input":"2023-07-14T17:54:38.122262Z","iopub.status.idle":"2023-07-14T17:54:38.131134Z","shell.execute_reply.started":"2023-07-14T17:54:38.122227Z","shell.execute_reply":"2023-07-14T17:54:38.129300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(tokenizer = lambda x: x.split(), binary=True)\ny_multilabel = vectorizer.fit_transform(prepared_df['tags'])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:09:08.221183Z","iopub.execute_input":"2023-07-14T18:09:08.221644Z","iopub.status.idle":"2023-07-14T18:09:08.361816Z","shell.execute_reply.started":"2023-07-14T18:09:08.221598Z","shell.execute_reply":"2023-07-14T18:09:08.359716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_multilabel.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:09:23.008662Z","iopub.execute_input":"2023-07-14T18:09:23.009559Z","iopub.status.idle":"2023-07-14T18:09:23.018560Z","shell.execute_reply.started":"2023-07-14T18:09:23.009503Z","shell.execute_reply":"2023-07-14T18:09:23.016379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(prepared_df.question, y_multilabel, test_size = 0.2,random_state = 42)\n\nprint(\"Number of data points in training data :\", X_train.shape[0])\nprint(\"Number of data points in test data :\", X_test.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:10:08.283694Z","iopub.execute_input":"2023-07-14T18:10:08.284192Z","iopub.status.idle":"2023-07-14T18:10:08.309008Z","shell.execute_reply.started":"2023-07-14T18:10:08.284155Z","shell.execute_reply":"2023-07-14T18:10:08.307906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:10:15.206535Z","iopub.execute_input":"2023-07-14T18:10:15.208345Z","iopub.status.idle":"2023-07-14T18:10:15.220175Z","shell.execute_reply.started":"2023-07-14T18:10:15.208279Z","shell.execute_reply":"2023-07-14T18:10:15.217798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:10:24.795375Z","iopub.execute_input":"2023-07-14T18:10:24.795891Z","iopub.status.idle":"2023-07-14T18:10:24.803348Z","shell.execute_reply.started":"2023-07-14T18:10:24.795864Z","shell.execute_reply":"2023-07-14T18:10:24.802273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Featurization of Training Data","metadata":{}},{"cell_type":"code","source":"def text_splitter(text):\n return text.split()\n\ntfidf = TfidfVectorizer(min_df=0.00009, max_features=200000, tokenizer = lambda x : x.split(), ngram_range=(1,4))\n\n\nX_train_multilabel = tfidf.fit_transform(X_train)\nX_test_multilabel = tfidf.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T19:06:07.753847Z","iopub.execute_input":"2023-07-14T19:06:07.754370Z","iopub.status.idle":"2023-07-14T19:06:23.828682Z","shell.execute_reply.started":"2023-07-14T19:06:07.754331Z","shell.execute_reply":"2023-07-14T19:06:23.826164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Training data shape X : \",X_train_multilabel.shape, \"Y :\",y_train.shape)\nprint(\"Test data shape X : \",X_test_multilabel.shape,\"Y:\",y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:46:53.230112Z","iopub.execute_input":"2023-07-14T18:46:53.230468Z","iopub.status.idle":"2023-07-14T18:46:53.237146Z","shell.execute_reply.started":"2023-07-14T18:46:53.230445Z","shell.execute_reply":"2023-07-14T18:46:53.235872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.reset_index().iloc[0,:]['question']","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:46:57.198783Z","iopub.execute_input":"2023-07-14T18:46:57.199249Z","iopub.status.idle":"2023-07-14T18:46:57.209679Z","shell.execute_reply.started":"2023-07-14T18:46:57.199217Z","shell.execute_reply":"2023-07-14T18:46:57.208512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 0. SGD Classifier using OVR","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:14:21.858552Z","iopub.execute_input":"2023-07-14T18:14:21.858950Z","iopub.status.idle":"2023-07-14T18:14:21.866514Z","shell.execute_reply.started":"2023-07-14T18:14:21.858925Z","shell.execute_reply":"2023-07-14T18:14:21.864950Z"}}},{"cell_type":"code","source":"from sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\n\nclf = OneVsRestClassifier(SGDClassifier(loss='log_loss', alpha=0.00001, penalty='l2'))\nclf.fit(X_train_multilabel, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:01.425664Z","iopub.execute_input":"2023-07-14T18:47:01.426123Z","iopub.status.idle":"2023-07-14T18:47:03.819964Z","shell.execute_reply.started":"2023-07-14T18:47:01.426074Z","shell.execute_reply":"2023-07-14T18:47:03.818958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ny_pred = clf.predict(X_test_multilabel)\n\nsgd_ovr_accuracy = accuracy_score(y_pred, y_test)\nsgd_ovr_macro_f1_score = metrics.f1_score(y_pred, y_test, average = 'macro')\nprint(f'SGD OVR accuracy {sgd_ovr_accuracy}')\nprint(f'SGD OVR Macro F1 Score: {sgd_ovr_macro_f1_score}')\n\nprint(classification_report(y_test, y_pred,target_names=list_tags))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:06.127529Z","iopub.execute_input":"2023-07-14T18:47:06.128967Z","iopub.status.idle":"2023-07-14T18:47:06.200915Z","shell.execute_reply.started":"2023-07-14T18:47:06.128919Z","shell.execute_reply":"2023-07-14T18:47:06.199842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Naive Bayes Classifier using OVR","metadata":{}},{"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB,BernoulliNB\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfTransformer,TfidfVectorizer\nfrom sklearn.svm import LinearSVC\n\n\nnb = OneVsRestClassifier(BernoulliNB())\nnb.fit(X_train_multilabel, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:41.630229Z","iopub.execute_input":"2023-07-14T18:47:41.630664Z","iopub.status.idle":"2023-07-14T18:47:42.173123Z","shell.execute_reply.started":"2023-07-14T18:47:41.630616Z","shell.execute_reply":"2023-07-14T18:47:42.171693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ny_pred = nb.predict(X_test_multilabel)\n\nnb_ovr_accuracy = accuracy_score(y_pred, y_test)\nnb_ovr_macro_f1_score = metrics.f1_score(y_pred, y_test, average = 'macro')\nprint(f'NB OVR accuracy {nb_ovr_accuracy}')\nprint(f'NB OVR Macro F1 Score: {nb_ovr_macro_f1_score}')\n\nprint(classification_report(y_test, y_pred,target_names=list_tags))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:46.700262Z","iopub.execute_input":"2023-07-14T18:47:46.701331Z","iopub.status.idle":"2023-07-14T18:47:46.951578Z","shell.execute_reply.started":"2023-07-14T18:47:46.701294Z","shell.execute_reply":"2023-07-14T18:47:46.950522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SGD Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\n\nsgd = OneVsRestClassifier(SGDClassifier(loss='hinge', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None))\nsgd.fit(X_train_multilabel, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:52.914678Z","iopub.execute_input":"2023-07-14T18:47:52.915862Z","iopub.status.idle":"2023-07-14T18:47:53.882542Z","shell.execute_reply.started":"2023-07-14T18:47:52.915793Z","shell.execute_reply":"2023-07-14T18:47:53.881700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ny_pred = sgd.predict(X_test_multilabel)\n\nsgd_ovr_hinge_accuracy = accuracy_score(y_pred, y_test)\nsgd_ovr_hinge_macro_f1_score = metrics.f1_score(y_pred, y_test, average = 'macro')\nprint(f'SGD OVR hinge accuracy {sgd_ovr_hinge_accuracy}')\nprint(f'SGD OVR hinge Macro F1 Score: {sgd_ovr_hinge_macro_f1_score}')\n\nprint(classification_report(y_test, y_pred,target_names=list_tags))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:47:56.029589Z","iopub.execute_input":"2023-07-14T18:47:56.030062Z","iopub.status.idle":"2023-07-14T18:47:56.086613Z","shell.execute_reply.started":"2023-07-14T18:47:56.030028Z","shell.execute_reply":"2023-07-14T18:47:56.084267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logistic Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlogreg = OneVsRestClassifier(LogisticRegression(n_jobs=1, C=1e5, solver='liblinear'))\nlogreg.fit(X_train_multilabel, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:48:03.498374Z","iopub.execute_input":"2023-07-14T18:48:03.498764Z","iopub.status.idle":"2023-07-14T18:48:18.987117Z","shell.execute_reply.started":"2023-07-14T18:48:03.498733Z","shell.execute_reply":"2023-07-14T18:48:18.985467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ny_pred = logreg.predict(X_test_multilabel)\n\nlogreg_ovr_accuracy = accuracy_score(y_pred, y_test)\nlogreg_ovr_macro_f1_score = metrics.f1_score(y_pred, y_test, average = 'macro')\nprint(f'LOGREG OVR accuracy {logreg_ovr_accuracy}')\nprint(f'LOGREG OVR Macro F1 Score: {logreg_ovr_macro_f1_score}')\n\nprint(classification_report(y_test, y_pred,target_names=list_tags))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:48:21.792879Z","iopub.execute_input":"2023-07-14T18:48:21.793309Z","iopub.status.idle":"2023-07-14T18:48:21.861076Z","shell.execute_reply.started":"2023-07-14T18:48:21.793277Z","shell.execute_reply":"2023-07-14T18:48:21.859273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_results = pd.DataFrame([{'Naive Bayes Classifier for Multinomial Models':nb_ovr_accuracy,\n 'SGD Classifier OVR log loss':sgd_ovr_accuracy,\n 'SGD Classifier OVR hinge loss':sgd_ovr_hinge_accuracy,\n 'Logistic regression':logreg_ovr_accuracy}]).T\ndf_results = df_results.rename({0:'Accuracy'}, axis=1)\ndf_results = df_results.sort_values(by='Accuracy', ascending=False)\n\ndisplay(df_results)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:48:31.859384Z","iopub.execute_input":"2023-07-14T18:48:31.859820Z","iopub.status.idle":"2023-07-14T18:48:31.873191Z","shell.execute_reply.started":"2023-07-14T18:48:31.859790Z","shell.execute_reply":"2023-07-14T18:48:31.871872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_results = pd.DataFrame([{'Naive Bayes Classifier for Multinomial Models':nb_ovr_macro_f1_score,\n 'SGD Classifier OVR log loss':sgd_ovr_macro_f1_score,\n 'SGD Classifier OVR hinge loss':sgd_ovr_hinge_macro_f1_score,\n 'Logistic regression':logreg_ovr_macro_f1_score}]).T\n\ndf_results = df_results.rename({0:'Macro F1 Score'}, axis=1)\ndf_results = df_results.sort_values(by='Macro F1 Score', ascending=False)\n\ndisplay(df_results)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:48:38.675172Z","iopub.execute_input":"2023-07-14T18:48:38.676016Z","iopub.status.idle":"2023-07-14T18:48:38.691216Z","shell.execute_reply.started":"2023-07-14T18:48:38.675971Z","shell.execute_reply":"2023-07-14T18:48:38.689221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"________________","metadata":{}},{"cell_type":"code","source":"tfidf","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:46:17.872161Z","iopub.execute_input":"2023-07-14T18:46:17.872678Z","iopub.status.idle":"2023-07-14T18:46:17.884124Z","shell.execute_reply.started":"2023-07-14T18:46:17.872642Z","shell.execute_reply":"2023-07-14T18:46:17.881676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\npickle.dump(tfidf,open('vectorizer.pkl','wb'))\npickle.dump(logreg,open('model.pkl','wb'))","metadata":{"execution":{"iopub.status.busy":"2023-07-14T18:53:00.620291Z","iopub.execute_input":"2023-07-14T18:53:00.620739Z","iopub.status.idle":"2023-07-14T18:53:02.010525Z","shell.execute_reply.started":"2023-07-14T18:53:00.620706Z","shell.execute_reply":"2023-07-14T18:53:02.008322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pprint(df_train.iloc[100,:].to_dict())","metadata":{"execution":{"iopub.status.busy":"2023-07-13T16:05:19.459441Z","iopub.execute_input":"2023-07-13T16:05:19.459809Z","iopub.status.idle":"2023-07-13T16:05:19.467644Z","shell.execute_reply.started":"2023-07-13T16:05:19.459783Z","shell.execute_reply":"2023-07-13T16:05:19.466454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unit testing","metadata":{}},{"cell_type":"code","source":"input_data = \"\"\"\nI have a website under development on next.js . a blog is already ready there. I am wondering if it is possible to make a blog of similar articles that are on many sites. The tag that is assigned to the article can serve as a filter. I will show my article component [slug.js]\n\nimport { getAllPosts, getPostBySlug } from \"@/lib/getAllData\";\nimport Seo from \"@/components/global/seo\";\nimport { serialize } from 'next-mdx-remote/serialize';\nimport { MDXRemote } from 'next-mdx-remote';\n\nimport Link from 'next/link'\nimport Date from '@/lib/date';\nimport styles from \"@/styles/home/posts.module.css\"\nimport PostFooter from \"@/components/extra/postFooter\"\n\nimport Button from '@/components/button';\n\nconst _ = require(\"lodash\")\nimport {translit} from \"@/lib/utils\"\n\nconst components = {\n Button\n}\n\nconst blogPostHeaderTopTagClasses = [styles.blogPostHeaderTopTag, \"mainBtn\"].join(\" \")\n\nconst getTagLink = (tag) => {\n return (\n \n {translit(tag)}\n \n );\n};\n\nconst posts = getAllPosts();\n\nexport async function getStaticPaths() {\n\n return {\n paths: posts.map((post) => {\n return {\n params: {\n slug: post.slug,\n },\n };\n }),\n fallback: false,\n };\n}\n\n\n export async function getStaticProps({ params: { slug } }) {\n const post = getPostBySlug(slug);\n const mdxSource = await serialize(post.content);\n\n const postIndex = posts.findIndex((post) => post.slug === slug)\n const prev = posts[postIndex + 1] || null\n const next = posts[postIndex - 1] || null\n\n return {\n props: {\n post,\n frontmatter: post.data,\n content: mdxSource,\n prev,\n next\n }\n };\n }\n\n\n export default function PostPage({ frontmatter, content, prev, next }) {\n\n const url = typeof window !== 'undefined' ? window.location.href : '';\n\n return (\n <>\n\n \n\n
\n
\n
\n
\n {frontmatter.tags.map(tag => getTagLink(tag)).reduce((prev, curr) => [prev, '', curr])}\n
\n

{frontmatter.title}

\n

{frontmatter.description}

\n \n \n \n
\n
\n
\n \n \n {(next || prev) && (\n
\n {prev && (\n
\n
\n Предыдущая статья\n
\n
\n {prev.title}\n
\n
\n )}\n {next && (\n
\n
\n Следующая статья\n
\n
\n {next.title}\n
\n
\n )}\n
\n )}\n
\n
\n \n );\n }\n\nLet's say embed the related posts block under the (next|| prev) block. I need to somehow match one of the tags that the currently open article has and other articles that have one of the same tags. Is the information provided by the code above enough, or is something else needed? Who will tell you what to do?\n\"\"\"\n\n\ndef striphtml(data):\n cleanr = re.compile('<.*?>')\n cleantext = re.sub(cleanr,' ',str(data))\n return cleantext\n\ndef pre_process(question):\n\n question=re.sub('(.*?)', '', question, flags=re.MULTILINE|re.DOTALL)\n question=striphtml(question.encode('utf-8'))\n question=re.sub(r'[^A-Za-z]+',' ',question)\n words=word_tokenize(str(question.lower()))\n\n #Removing all single letter and and stopwords from question exceptt for the letter 'c'\n question=' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))\n\n return question\n\n\npreprocessed = pre_process(input_data)\n\npreprocessed = [word for word in preprocessed.split()]\n\nprint(preprocessed)\n\nvect_input = tfidf.transform(preprocessed)\n\ny_pred = logreg.predict(vect_input)\n\ny_pred_proba = logreg.predict_proba(vect_input)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T20:07:04.969275Z","iopub.execute_input":"2023-07-14T20:07:04.969701Z","iopub.status.idle":"2023-07-14T20:07:05.009374Z","shell.execute_reply.started":"2023-07-14T20:07:04.969670Z","shell.execute_reply":"2023-07-14T20:07:05.007640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred.toarray()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T20:07:06.561142Z","iopub.execute_input":"2023-07-14T20:07:06.561824Z","iopub.status.idle":"2023-07-14T20:07:06.570918Z","shell.execute_reply.started":"2023-07-14T20:07:06.561788Z","shell.execute_reply":"2023-07-14T20:07:06.569158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_list = tag_df_train_sorted['Tags'][:20].to_list()","metadata":{"execution":{"iopub.status.busy":"2023-07-14T20:07:07.863070Z","iopub.execute_input":"2023-07-14T20:07:07.863539Z","iopub.status.idle":"2023-07-14T20:07:07.871152Z","shell.execute_reply.started":"2023-07-14T20:07:07.863505Z","shell.execute_reply":"2023-07-14T20:07:07.869584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"freq_dict = {}\nwords = np.where(y_pred.toarray() == 1)\n\nfor items in words:\n if output_list[items[0]] in freq_dict:\n freq_dict[output_list[items[0]]] += 1\n else: \n freq_dict[output_list[items[0]]] = 1\n \nsorted_items = sorted(freq_dict.items(), key=lambda x: x[1], reverse=True)\n\n# Get the top 3 elements from the sorted list\ntop_3_elements = sorted_items[:3]\n\n# Print the top 3 elements\nfor element, count in top_3_elements:\n print(element)","metadata":{"execution":{"iopub.status.busy":"2023-07-14T20:07:18.837808Z","iopub.execute_input":"2023-07-14T20:07:18.838512Z","iopub.status.idle":"2023-07-14T20:07:18.849615Z","shell.execute_reply.started":"2023-07-14T20:07:18.838469Z","shell.execute_reply":"2023-07-14T20:07:18.847842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-14T19:53:01.112916Z","iopub.execute_input":"2023-07-14T19:53:01.114424Z","iopub.status.idle":"2023-07-14T19:53:01.121408Z","shell.execute_reply.started":"2023-07-14T19:53:01.114356Z","shell.execute_reply":"2023-07-14T19:53:01.119565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pprint(df_train.iloc[2].to_dict())","metadata":{"execution":{"iopub.status.busy":"2023-07-14T19:17:05.301485Z","iopub.execute_input":"2023-07-14T19:17:05.301917Z","iopub.status.idle":"2023-07-14T19:17:05.309618Z","shell.execute_reply.started":"2023-07-14T19:17:05.301883Z","shell.execute_reply":"2023-07-14T19:17:05.307847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.iloc[2]['Body']","metadata":{"execution":{"iopub.status.busy":"2023-07-14T19:19:14.397868Z","iopub.execute_input":"2023-07-14T19:19:14.398808Z","iopub.status.idle":"2023-07-14T19:19:14.408345Z","shell.execute_reply.started":"2023-07-14T19:19:14.398772Z","shell.execute_reply":"2023-07-14T19:19:14.406209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}