{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1) Libraries Used","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd # for operations on csv\nimport numpy as np # Just love to import this, comes handy sometimes\nimport seaborn as sns # for plotting\nimport matplotlib.pyplot as plt # for plotting\nfrom sklearn.feature_extraction.text import CountVectorizer #To convert to BOW representation\nfrom sklearn.feature_extraction.text import TfidfVectorizer #To convert to Tf-IDF representation\nfrom nltk.corpus import stopwords #used to remove words in NLP which does not provide contextual information like ('will','is') etc.\nfrom wordcloud import WordCloud # for printing wordcloud\nfrom nltk.stem.snowball import SnowballStemmer # for getting the root of the words\nfrom sklearn.model_selection import train_test_split# for separating data into test/train,this way it is faster and more efficient\nfrom sklearn.multiclass import OneVsRestClassifier# used for multi label classification\nimport re # used for filtering regular expressions\nfrom nltk.tokenize import word_tokenize # used to convert words into tokens,equal to lamba x:x.split()\nfrom sklearn.linear_model import SGDClassifier #ML Model\nfrom sklearn.linear_model import LogisticRegression# ML Model\nfrom sklearn import metrics # Metrics used for evaluating model\nfrom sklearn.metrics import f1_score,precision_score,recall_score # Metrics used for evaluating model\nfrom sklearn.feature_extraction.text import TfidfVectorizer # Create a TF-IDF vector\nfrom bs4 import BeautifulSoup # used here for Removing Http tags like <b>\nfrom datetime import datetime # used to calculate time running a block\nfrom scipy.sparse import hstack # used to add dense array to sparse matrix for train both of them together\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2) Train Data Overview ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/facebook-recruiting-iii-keyword-extraction/Train.zip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total Rows: {}\\nTotal Columns: {}'.format(data.shape[0],data.shape[1]))\nprint('Columns: ',data.columns[0],data.columns[1],data.columns[2],data.columns[3])\nprint('Size of Train.zip in GB \\'{}\\' '.format(os.path.getsize('/kaggle/input/facebook-recruiting-iii-keyword-extraction/Train.zip')/10**9))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('ML Problem: We need to predict Tags from Title and Body, also we have multiple tags to predict for a single data point, this is a multi-label Classification problem')\nprint('\\n\\nRefer: https://www.analyticsvidhya.com/blog/2017/08/introduction-to-multi-label-classification/  to understand how to solve multi label classification')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3) Performance metric Used\n* Hamming Loss(Used in multi-label classification):https://www.kaggle.com/wiki/HammingLoss \n* Micro F1 Score(Used in multi label classification):https://www.kaggle.com/wiki/MeanFScore \n* Macro F1 Score(Used in multi label classification):https://www.kaggle.com/wiki/MeanFScore ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 4) Exploratory Data Analysis\nI will take less data to perform EDA and also to solve ML problem, as I don't have much computational power available on kaggle","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.iloc[:10000,:]\nprint(data.shape)\nprint(data.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.1) Checking for Duplicates","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_copy = data.copy()\nduplicates = data_copy.duplicated(['Body','Tags'])\nprint('Total Duplicates:',duplicates.sum())\nprint('-'*50,'Removing Duplicates','-'*50)\n\ndata_copy = data_copy[~duplicates]\n\nprint('Shape of Data Frame Now',data_copy.shape)\nprint('-'*30,'Checking for any More Duplicates','-'*30)\nprint('Any More Duplicated data ', data_copy.duplicated(subset=['Title','Tags']).sum())\nprint('-'*30,'Analysing the Duplicated Data','-'*30)\n\n\nprint('\\nStill Duplicated ID:')\n#print( data_copy[data_copy['Title'] == data_copy['Title']]['Tags'])\nprint(data_copy[data_copy.duplicated(subset=['Title','Tags'])]['Id'])\nprint('----'*70)\nprint(data_copy[data_copy['Id']==3081]['Body'])\nprint('----'*70)\nprint(data_copy[data_copy['Id']==3082]['Body'])\nprint('----'*70)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Body and tags are completely different, so no further duplictes')\n\nprint(data_copy.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.2) Analysis of Tags","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"##Creating new column for tags count\ndata_copy['Tag_Count'] = data_copy['Tags'].apply(lambda x: len(x.split(' '))) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Distribution of Tags per question:\\n',data_copy['Tag_Count'].value_counts() )\nsns.countplot(data_copy['Tag_Count'])\nplt.ylabel('Number of question')\nplt.title('Distribution of Tags per question')\nplt.show()\nprint('Average Distribution of Tags',data_copy['Tag_Count'].sum()/len(data_copy['Tag_Count']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Observation\n* Average Distribution of Tags 2.888603462577935\n* 3 and 2 count of Tags dominate the dataset\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"##Converting Tags to BOW representation\nvectorizer = CountVectorizer() #Default tokenizer = lamba x:x.split()\ntag_data = vectorizer.fit_transform(data_copy['Tags'])\nprint('Sparse Matrix BOW Rows/Total Data Points:',tag_data.shape[0])\nprint('Sparse Matrix BOW Columns/Total Uniqe Tags:',tag_data.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Used to get the column/feature information from sparse matrix.\ntag = vectorizer.get_feature_names()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## https://stackoverflow.com/questions/15115765/how-to-access-sparse-matrix-elements\nfrequency = tag_data.sum(axis=0)\nfrequency = np.array(frequency)\n\ntemp = zip(tag,frequency[0])\ntag_count = dict(temp)\n\nkey =[]\nvalue=[]\nfor keys,values in tag_count.items():\n    key.append(keys)\n    value.append(values)\n\ntag_count_dataframe = pd.DataFrame(key)\ntag_count_dataframe['Count'] = value\ntag_count_dataframe = tag_count_dataframe.rename(columns ={0:'Tag'})\ntag_count_dataframe['TAG ID'] = tag_count_dataframe.index +1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_count_dataframe = tag_count_dataframe.sort_values(['Count'],ascending='False')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_count_dataframe.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_axes([0,0,1.5,1.5])\nax.plot(tag_count_dataframe['Count'].values)\nax.set_ylabel('Tag Appearance Count')\nax.set_xlabel('Tag ID')\nax.set_title(\"Distribution of number of times tag appeared questions\")\n\nax2 = fig.add_axes([0.15,0.5,.4,.4])\nax2.plot(tag_count_dataframe['Count'].values[4417:4917])\nax2.set_ylabel('Tag Appearance Count')\nax2.set_xlabel('Tag ID')\nax2.set_title(\"Zoom for last 500 tags\")\n\nax3 = fig.add_axes([0.8,0.5,.4,.4])\nax3.plot(tag_count_dataframe['Count'].values[4817:4917])\nax3.set_ylabel('Tag Appearance Count')\nax3.set_xlabel('Tag ID')\nax3.set_title(\"Zooming further for last 100 tags\")\nax.annotate('', xy=(4417, 1), xytext=(1500, 300),\n            arrowprops=dict(facecolor='black', shrink=0.05),\n            )\n\nax.annotate('', xy=(4817, 30), xytext=(4000, 300),\n            arrowprops=dict(facecolor='black', shrink=0.05),\n            )\nfig.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('I will take limited amount of tags, as with increase of each tag I will have to train one extra model, which is computationlly expensive for me')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.3) Fancy Wordcloud","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud = WordCloud(background_color='black',width=1600,height=800).generate_from_frequencies(tag_count)\nfig = plt.figure(figsize=(30,20))\nplt.imshow(wordcloud)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.4) Top 20 Tags","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.barplot(tag_count_dataframe['Tag'].values[4897:4917],tag_count_dataframe['Count'].values[4897:4917],color='red')\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Observation\n* net,Java and PHP are most popular tags","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 5) Pre-Processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"stop_words=set(stopwords.words('english'))\nstemmer = SnowballStemmer(\"english\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_list  = []\nquestion_with_code = [ ]\nlen_before_preprocessing = 0 \nlen_after_preprocessing = 0 \ntime_before=datetime.now()\n\ndata_copy = data_copy.drop(['Id'],axis=1)\n\nfor _,rows in data_copy.iterrows():\n    title,body,tags = rows[0],rows[1],rows[2]\n    if '<code>' in body:\n        question_with_code.append(1)\n    else:\n        question_with_code.append(0)\n    len_before_preprocessing += len(title) + len(body)\n    body = re.sub('<code>(.*?)<code>', ' ', body, flags = re.DOTALL|re.MULTILINE)\n    body = re.sub('<.*?>',' ',body)\n    body = body.encode('utf-8')\n    title = title.encode('utf-8')\n    question = str(title) + ' ' + str(title) + ' ' + str(title) + ' ' + str(body)## Giving 3 times more importance to title\n    question = re.sub(r'[^A-Za-z]+',' ',question) # removes punctuation marks or special characters\n    question = re.sub(r'http\\S+',' ',question) # removes all http tags\n    soup = BeautifulSoup(question, 'lxml') # removes all xml tags\n    question = soup.get_text()\n    words=word_tokenize(str(question.lower()))\n    question = ' '.join(str(stemmer.stem(i)) for i in words if i not in stop_words and (len(i)>1 or i=='c'))\n    question_list.append(question)\n    len_after_preprocessing += len(question)\ndata_copy['question'] = question_list\ndata_copy['code_exist'] = question_with_code\ntime_after = datetime.now()\navg_len_before_preprocessing=(len_before_preprocessing*1.0)/data_copy.shape[0]\navg_len_after_preprocessing=(len_after_preprocessing*1.0)/data_copy.shape[0]\nprint( \"Avg. length of questions(Title+Body) before preprocessing: \", avg_len_before_preprocessing)\nprint( \"Avg. length of questions(Title+Body) after preprocessing: \", avg_len_after_preprocessing)\nprint (\"% of questions containing code: \", (sum(question_with_code)*100.0)/data_copy.shape[0])\nprint('Time taken to run this cell',time_after-time_before)\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_df = data_copy.drop(['Title','Body','Tag_Count'],axis=1)\npreprocessed_df.to_csv('Preprocessed_data_Stackoverflow.csv')\nprint(\"Shape of preprocessed data :\", preprocessed_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6) Machine Learning problem","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"6.1) Creating Multilabels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = CountVectorizer(tokenizer = lambda x: x.split(), binary='true') # binary=True--> One hot Enoding without removing first column in this case\ny_multilabel = vectorizer.fit_transform(preprocessed_df['Tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tags_to_consider(n):\n    tag_i_sum = y_multilabel.sum(axis=0).tolist()[0]\n    sorted_tags_i = sorted(range(len(tag_i_sum)), key=lambda i: tag_i_sum[i], reverse=True)\n    yn_multilabel=y_multilabel[:,sorted_tags_i[:n]]\n    return yn_multilabel\n\ndef questions_covered_fn(numb):\n    yn_multilabel = tags_to_consider(numb)\n    x= yn_multilabel.sum(axis=1)\n    return (np.count_nonzero(x==0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_covered = []\ntotal_tags=y_multilabel.shape[1]\ntotal_qus=preprocessed_df.shape[0]\nfor i in range(100, total_tags, 100):\n    questions_covered.append(np.round(((total_qus-questions_covered_fn(i))/total_qus)*100,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.arange(100,total_tags, 100),questions_covered)\nplt.xlabel(\"Number of tags\")\nplt.ylabel(\"Number of questions covered partially\")\nplt.grid()\nplt.show()\nprint(questions_covered[9],\"% of questions covered by 1000 tags\")\nprint(\"Number of questions that are not covered by 100 tags : \", questions_covered_fn(1000),\"out of \", total_qus)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yx_multilabel = tags_to_consider(1000)\nprint(\"Number of tags in the subset :\", y_multilabel.shape[1])\nprint(\"Number of tags considered :\", yx_multilabel.shape[1],\"(\",(yx_multilabel.shape[1]/y_multilabel.shape[1])*100,\"%)\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"7.2) SPlitting into test and train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(preprocessed_df, yx_multilabel, test_size = 0.2,random_state = 42)\nprint(\"Number of data points in training data :\", X_train.shape[0])\nprint(\"Number of data points in test data :\", X_test.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = TfidfVectorizer(min_df=0.00009, max_features=200000, tokenizer = lambda x: x.split(), ngram_range=(1,3))\nX_train_multilabel = vectorizer.fit_transform(X_train['question'])\nX_test_multilabel = vectorizer.transform(X_test['question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training data shape X : \",X_train_multilabel.shape, \"Y :\",y_train.shape)\nprint(\"Test data shape X : \",X_test_multilabel.shape,\"Y:\",y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"7.3) Fitting logistic and One vs Rest classifier and adding 'code_exist' column to my train and test data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.sparse import hstack ## Adding code_exist column to my training data\nX_train_multilabel = hstack((X_train_multilabel,np.array(X_train['code_exist'])[:,None]))\nX_test_multilabel = hstack((X_test_multilabel,np.array(X_test['code_exist'])[:,None]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_before=datetime.now()\n\nclf = OneVsRestClassifier(SGDClassifier(loss='log', alpha=0.00001, penalty='l1'))\nclf.fit(X_train_multilabel, y_train)\ny_pred = clf.predict(X_test_multilabel)\ntime_after = datetime.now()\nprint('Time taken to run this cell',time_after-time_before)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Accuracy :\",metrics.accuracy_score(y_test,y_pred))\nprint(\"Macro f1 score :\",metrics.f1_score(y_test, y_pred, average = 'macro'))\nprint(\"Micro f1 scoore :\",metrics.f1_score(y_test, y_pred, average = 'micro'))\nprint(\"Hamming loss :\",metrics.hamming_loss(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}