{"cells":[{"metadata":{},"cell_type":"markdown","source":"# StackOverflow tag prediction\n\n*  This notebook has two parts, in 1st part I've used SGD Classifier and in 2nd part I've used BiLSTM"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom tqdm import tqdm\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS\n# from jupyterthemes import jtplot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.preprocessing import StandardScaler\n\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix, classification_report\n# from sklearn.metrics import precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\n\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.snowball import SnowballStemmer\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk import ngrams\n\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.models import Sequential\nfrom keras.layers import LSTM, Dense, TimeDistributed, Bidirectional, Embedding, Dropout, Flatten\nfrom keras.optimizers import SGD, Adam, Adagrad, Adadelta, RMSprop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set plot rc parameters\n\n# jtplot.style(grid=False)\nplt.rcParams['figure.facecolor'] = 'white'\nplt.rcParams['axes.facecolor'] = '#464646'\n#plt.rcParams['axes.edgecolor'] = '#FFFFFF'\nplt.rcParams['figure.figsize'] = 10, 7\nplt.rcParams['text.color'] = '#666666'\nplt.rcParams['axes.labelcolor'] = '#666666'\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['axes.titlesize'] = 16\nplt.rcParams['xtick.color'] = '#666666'\nplt.rcParams['xtick.labelsize'] = 14\nplt.rcParams['ytick.color'] = '#666666'\nplt.rcParams['ytick.labelsize'] = 14\n\n# plt.rcParams['font.size'] = 16\n\nsns.color_palette('dark')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/facebook-recruiting-iii-keyword-extraction/Train.zip', usecols=['Id', 'Title', 'Tags'])\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop_duplicates('Title', inplace=True)\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA"},{"metadata":{},"cell_type":"markdown","source":"### Tag Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get number of tags for each title\ntrain['Tag_count'] = train['Tags'].apply(lambda x: len(str(x).split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dropna()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[~train['Tags'].isnull()]\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Distribution of tag count"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot distribution of tag count\nfig = plt.figure(figsize=[10,7])\nsns.countplot(train['Tag_count'])\nplt.title('Distribution of tag count')\nplt.ylabel('Frequency')\nplt.xlabel('Tag count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Get list of all tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# vectorize tags\ntag_vectorizer = CountVectorizer(tokenizer= lambda x: str(x).split())\ntag_mat = tag_vectorizer.fit_transform(train['Tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get names of tags\ntag_names = tag_vectorizer.get_feature_names()\ntype(tag_names), len(tag_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_names[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get frequency of each tag\ntag_freq = tag_mat.sum(axis=0)\ntype(tag_freq), tag_freq.A1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# store tag names and frequency as a pandas series\ntag_freq_ser = pd.Series(tag_freq.A1, index=tag_names)\ntag_freq_ser.sort_values(ascending=False, inplace=True)\ntag_freq_ser.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Histogram of tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot distribution of tag frequency\nfig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*  Hard to figure out anything\n*  let's plot top 500 Tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot distribution of tag frequency (top 500)\nfig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.iloc[:500].values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution of top 500 Tags')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot distribution of tag frequency (top 100)\nfig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.iloc[:100].values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution of top 100 Tags')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Top words used as tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot word count for tags\nwordcloud = WordCloud(background_color='black',\n                      max_words=200).generate_from_frequencies(tag_freq_ser)\nfig = plt.figure(figsize=[16,16])\nplt.title('WordCloud of Tags')\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Frequency of top 30 tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot top 30 tags\nfig = plt.figure(figsize=[20,10])\nsns.barplot(x=tag_freq_ser.iloc[:50].index,\n            y=tag_freq_ser.iloc[:50].values,\n           color=sns.xkcd_rgb['greenish cyan'])\nplt.title('Frequency of top 50 Tags')\nplt.xlabel('Tags')\nplt.ylabel('Frequency')\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Part 1"},{"metadata":{},"cell_type":"markdown","source":"## Data Pre-processing"},{"metadata":{},"cell_type":"markdown","source":"### Clean text data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# clean text data\n# remove non alphabetic characters\n# remove stopwords and stemming\ndef clean_text(sentence):\n    # remove non alphabetic sequences\n    pattern = re.compile(r'[^a-z]+')\n    sentence = sentence.lower()\n    sentence = pattern.sub(' ', sentence).strip()\n    \n    # Tokenize\n    word_list = word_tokenize(sentence)\n    # stop words\n    stopwords_list = set(stopwords.words('english'))\n    # remove stop words\n    word_list = [word for word in word_list if word not in stopwords_list]\n    # stemming\n    ps  = PorterStemmer()\n    word_list = [ps.stem(word) for word in word_list]\n    # list to sentence\n    sentence = ' '.join(word_list)\n    \n    return sentence\n\n# clean text data\ntqdm.pandas()\ntrain['Title'] = train['Title'].progress_apply(lambda x: clean_text(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reduce number of tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate number of questions covered by top n tags\ndef questions_covered(one_hot_tag, ntags):\n    # number of questions\n    nq = one_hot_tag.shape[0]\n    # get number of questions covered by each tag\n    tag_sum = one_hot_tag.sum(axis=0).tolist()[0]\n    # sort tags based on number of questions covered by them\n    tag_sum_sorted = sorted(range(len(tag_sum)),\n                            key=lambda x: tag_sum[x],\n                            reverse=True)\n    # get one hot encoded matrix for top n tags\n    one_hot_topn_tag = one_hot_tag[:, tag_sum_sorted[:ntags]]\n    # get number of tags per question\n    tags_per_question = one_hot_topn_tag.sum(axis=1)\n    # get number of question with no tags\n    q_with_0_tags = np.count_nonzero(tags_per_question == 0)\n    \n    return np.round((nq - q_with_0_tags)/nq*100, 2)\n\n# get number of questions covered and tag id list\ndef questions_covered_list(one_hot_tag, window):\n    # number of tags\n    ntags = one_hot_tag.shape[1]\n    # question id list\n    qid_list = np.arange(100, ntags, window)\n    # questions covered list\n    ques_covered_list = []\n    for idx in range(100, ntags, window):\n        ques_covered_list.append(questions_covered(one_hot_tag, idx))\n        \n    return qid_list, ques_covered_list\n\n# get multinomial tag matrix (top n tags)\ndef topn_tags(one_hot_tag, ntags):\n    # get number of questions covered by each tag\n    tag_sum = one_hot_tag.sum(axis=0).tolist()[0]\n    # sort tags based on number of questions covered by them\n    tag_sum_sorted = sorted(range(len(tag_sum)),\n                            key=lambda x: tag_sum[x],\n                            reverse=True)\n    # get one hot encoded matrix for top n tags\n    one_hot_topn_tag = one_hot_tag[:, tag_sum_sorted[:ntags]]\n    return one_hot_topn_tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using bag of words to represent tags for each title\ntag_vectorizer = CountVectorizer(tokenizer= lambda x: str(x).split(), binary=True)\ny_multinomial = tag_vectorizer.fit_transform(train['Tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = questions_covered_list(y_multinomial, 100)\nfig = plt.figure(figsize=[10,7])\nplt.title('Questions covered Vs Numbre of Tags')\nplt.ylabel('Percentage of Questions covered')\nplt.xlabel('Number of Tags')\nplt.plot(x,y, c=sns.xkcd_rgb['greenish cyan'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print percent of question covered with number of tags\nprint('#Tags\\t%Ques')\nfor idx in range(500, 7500, 500):\n    print(idx, '\\t', y[int(idx/100)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_multinomial = topn_tags(y_multinomial, 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get index of questions covered\n# and remove rest of the data\nnon_zero_idx = y_multinomial.sum(axis=1) != 0\nnon_zero_idx = non_zero_idx.A1\ny_multinomial = y_multinomial[non_zero_idx,:]\ntrain = train.iloc[non_zero_idx, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_multinomial.shape, train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Featurize data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# split data in 80-20 ratio\nXtrain, Xtest, Ym_train, Ym_test = train_test_split(train['Title'], y_multinomial, test_size=0.2, random_state=45)\n\n# vectorize text data\ntfid_vec = TfidfVectorizer(tokenizer=lambda x: str(x).split())\nXtrain = tfid_vec.fit_transform(Xtrain)\nXtest = tfid_vec.transform(Xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain.shape, Xtest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ym_train.shape, Ym_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SGDClassifier one vs rest"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create model instance\nlogreg_model1 = OneVsRestClassifier(SGDClassifier(loss='log',\n                                                  alpha=0.001,\n                                                  penalty='l1'),\n                                   n_jobs=-1)\n# train model\nlogreg_model1.fit(Xtrain, Ym_train)\n# predict tags\nYm_test_pred = logreg_model1.predict(Xtest)\n\n# print model performance metrics\nprint(\"Accuracy :\",metrics.accuracy_score(Ym_test,Ym_test_pred))\nprint(\"f1 score macro :\",metrics.f1_score(Ym_test,Ym_test_pred, average = 'macro'))\nprint(\"f1 scoore micro :\",metrics.f1_score(Ym_test,Ym_test_pred, average = 'micro'))\nprint(\"Hamming loss :\",metrics.hamming_loss(Ym_test,Ym_test_pred))\n# print(\"Precision recall report :\\n\",metrics.classification_report(Ym_test,Ym_test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"*  Kaggle kernels doesn't have sufficient memory to train 5000 (tags to predict) models\n*  I trained 100 models and got results shown above, which are not that impressive\n*  If you guys have good machine you can try to train 5000 models on it and check their performance\n*  you can also try other models like logistic regression, SVM, randomforest etc."},{"metadata":{},"cell_type":"markdown","source":"## Part 2"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Bidirectional LSTM"},{"metadata":{},"cell_type":"markdown","source":"### Data Prepration\n\n*  Let's just prepare our X variable \n*  we can use y variable generated in part 1\n*  Data is already clean we just need to generate word to number and vice versa\n*  use that map to encode word sequences and then embed them"},{"metadata":{"trusted":true},"cell_type":"code","source":"# tokenize words in title\nt = Tokenizer(num_words=20000)\nt.fit_on_texts(train['Title'].to_list())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# word to number and vice versa map\nw2num = t.word_index\nnum2w = {k:w for w, k in w2num.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# replace words with numbers in docs\ndocs = train['Title'].to_list()\ndocs2 = []\nfor doc in docs:\n    \n    lst = []\n    \n    for word in doc.split():\n        lst.append(w2num[word])\n        \n    docs2.append(lst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pad sequences\ndocs2 = pad_sequences(docs2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain, Xcv, Ytrain, Ycv = train_test_split(docs2, y_multinomial, random_state=21, test_size=0.25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Initialize model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# initialize BiLSTM model\nmodel = Sequential()\n# embedding layer\nmodel.add(Embedding(20000, 256, input_length=27))\n# BiLSTM layer 1\nmodel.add(Bidirectional(LSTM(256, return_sequences=True)))\nmodel.add(Dropout(0.4))\n# BiLSTM layer 2\nmodel.add(Bidirectional(LSTM(256, return_sequences=True)))\nmodel.add(Dropout(0.4))\nmodel.add(Flatten())\nmodel.add(Dense(100, activation='sigmoid'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n# train LSTM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(Xtrain, Ytrain, epochs=10, batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict tags\nYcv_pred = model.predict(Xcv)\nYcv_pred = (Ycv_pred > 0.5).astype('int64')\n# print model performance metrics\nprint(\"Accuracy :\",metrics.accuracy_score(Ycv,Ycv_pred))\nprint(\"f1 score macro :\",metrics.f1_score(Ycv,Ycv_pred, average = 'macro'))\nprint(\"f1 scoore micro :\",metrics.f1_score(Ycv,Ycv_pred, average = 'micro'))\nprint(\"Hamming loss :\",metrics.hamming_loss(Ycv,Ycv_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*  Results are better compared to SGD but not good enough\n*  To make it better first thing we need is a good machine\n*  Next we can train for more epochs\n*  If you guys have patience and resources you can also try to include \"body\" of the query, here I've just used titles\n*  Another thing to try out is BiLSTM with attnetion, that might help a lot"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}