{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sqlite3\nimport re\nfrom tqdm import tqdm\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.cluster import KMeans\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\n#from sklearn.linear_model import LogisticRegression\nfrom skmultilearn.adapt import MLkNN\nfrom sklearn.metrics import f1_score, precision_score, recall_score, silhouette_score\n\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.snowball import SnowballStemmer\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk import ngrams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.facecolor'] = 'white'\nplt.rcParams['axes.facecolor'] = '#464646'\nplt.rcParams['figure.figsize'] = 10, 7\nplt.rcParams['text.color'] = '#666666'\nplt.rcParams['axes.labelcolor'] = '#666666'\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['axes.titlesize'] = 16\nplt.rcParams['xtick.color'] = '#666666'\nplt.rcParams['xtick.labelsize'] = 14\nplt.rcParams['ytick.color'] = '#666666'\nplt.rcParams['ytick.labelsize'] = 14\n\nsns.color_palette('dark')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/facebook-recruiting-iii-keyword-extraction/Train.zip')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.iloc[10000:30000, :]\n\nprint(\"Shape of training dataframe after subsetting : \", train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Tag_count'] = train['Tags'].apply(lambda x: len(str(x).split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dropna()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[~train['Tags'].isnull()]\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=[10,7])\nsns.countplot(train['Tag_count'])\nplt.title('Distribution of tag count')\nplt.ylabel('Frequency')\nplt.xlabel('Tag count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_vectorizer = CountVectorizer(tokenizer= lambda x: str(x).split())\ntag_mat = tag_vectorizer.fit_transform(train['Tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_names = tag_vectorizer.get_feature_names()\ntype(tag_names), len(tag_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_freq = tag_mat.sum(axis=0)\ntype(tag_freq), tag_freq.A1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_freq_ser = pd.Series(tag_freq.A1, index=tag_names)\ntag_freq_ser.sort_values(ascending=False, inplace=True)\ntag_freq_ser.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.iloc[:500].values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution of top 500 Tags')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=[10,7])\nplt.plot(tag_freq_ser.iloc[:100].values,\n         c=sns.xkcd_rgb['greenish cyan'])\nplt.title('Tag frequency distribution of top 100 Tags')\nplt.ylabel('Frequency')\nplt.xlabel('Tag ID')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud = WordCloud(background_color='blue',\n                      max_words=250).generate_from_frequencies(tag_freq_ser)\nfig = plt.figure(figsize=[15,15])\nplt.title('WordCloud of Tags')\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=[20,10])\nsns.barplot(x=tag_freq_ser.iloc[:50].index,\n            y=tag_freq_ser.iloc[:50].values,\n           color=sns.xkcd_rgb['blue'])\nplt.title('Frequency of top 50 Tags')\nplt.xlabel('Tags')\nplt.ylabel('Frequency')\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Data Preprocessing**"},{"metadata":{},"cell_type":"markdown","source":"Cleaning text data"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(sentence):\n    pattern = re.compile(r'[^a-z]+')\n    sentence = sentence.lower()\n    sentence = pattern.sub(' ', sentence).strip()\n    \n    word_list = word_tokenize(sentence)\n    stopwords_list = set(stopwords.words('english'))\n    word_list = [word for word in word_list if word not in stopwords_list]\n    ps  = PorterStemmer()\n    word_list = [ps.stem(word) for word in word_list]\n    sentence = ' '.join(word_list)\n    \n    return sentence\n\ntqdm.pandas()\ntrain['Title'] = train['Title'].progress_apply(lambda x: clean_text(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def questions_covered(one_hot_tag, ntags):\n    nq = one_hot_tag.shape[0]\n    tag_sum = one_hot_tag.sum(axis=0).tolist()[0]\n    tag_sum_sorted = sorted(range(len(tag_sum)),\n                            key=lambda x: tag_sum[x],\n                            reverse=True)\n    one_hot_topn_tag = one_hot_tag[:, tag_sum_sorted[:ntags]]\n    tags_per_question = one_hot_topn_tag.sum(axis=1)\n    q_with_0_tags = np.count_nonzero(tags_per_question == 0)\n    \n    return np.round((nq - q_with_0_tags)/nq*100, 2)\n\ndef questions_covered_list(one_hot_tag, window):\n    ntags = one_hot_tag.shape[1]\n    qid_list = np.arange(100, ntags, window)\n    ques_covered_list = []\n    for idx in range(100, ntags, window):\n        ques_covered_list.append(questions_covered(one_hot_tag, idx))\n        \n    return qid_list, ques_covered_list\n\n\ndef topn_tags(one_hot_tag, ntags):\n    tag_sum = one_hot_tag.sum(axis=0).tolist()[0]\n    tag_sum_sorted = sorted(range(len(tag_sum)),\n                            key=lambda x: tag_sum[x],\n                            reverse=True)\n    one_hot_topn_tag = one_hot_tag[:, tag_sum_sorted[:ntags]]\n    return one_hot_topn_tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_vectorizer = CountVectorizer(tokenizer= lambda x: str(x).split(), binary=True)\ny_multinomial = tag_vectorizer.fit_transform(train['Tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = questions_covered_list(y_multinomial, 100)\nfig = plt.figure(figsize=[10,7])\nplt.title('Questions covered Vs Numbre of Tags')\nplt.ylabel('Percentage of Questions covered')\nplt.xlabel('Number of Tags')\nplt.plot(x,y, c=sns.xkcd_rgb['greenish cyan'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#Tags\\t%Ques')\nfor idx in range(500, 7500, 500):\n    print(idx, '\\t', y[int(idx/100)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_multinomial = topn_tags(y_multinomial, 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_zero_idx = y_multinomial.sum(axis=1) != 0\nnon_zero_idx = non_zero_idx.A1\ny_multinomial = y_multinomial[non_zero_idx,:]\ntrain = train.iloc[non_zero_idx, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_multinomial.shape, train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain, Xtest, Ym_train, Ym_test = train_test_split(train['Title'], y_multinomial, test_size=0.2, random_state=45)\n\ntfid_vec = TfidfVectorizer(tokenizer=lambda x: str(x).split())\nXtrain = tfid_vec.fit_transform(Xtrain)\nXtest = tfid_vec.transform(Xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain.shape, Xtest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ym_train.shape, Ym_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_score(y_test, y_pred):\n    print(\"Accuracy :\",metrics.accuracy_score(y_test,y_pred))\n    print(\"Macro f1 score :\",metrics.f1_score(y_test, y_pred, average = 'macro'))\n    print(\"Micro f1 scoore :\",metrics.f1_score(y_test, y_pred, average = 'micro'))\n    print(\"Hamming loss :\",metrics.hamming_loss(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Supervised Model"},{"metadata":{},"cell_type":"markdown","source":"SGDClassifier one vs rest"},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\n\nstart = datetime.now()\n# create model instance\nlogreg_model1 = OneVsRestClassifier(SGDClassifier(loss='log',\n                                                  alpha=0.001,\n                                                  penalty='l1'),\n                                   n_jobs=-1)\n# train model\nlogreg_model1.fit(Xtrain, Ym_train)\n# predict tags\nYm_test_pred = logreg_model1.predict(Xtest)\n\n# print model performance metrics\nprint(\"Accuracy :\",metrics.accuracy_score(Ym_test,Ym_test_pred))\nprint(\"f1 score macro :\",metrics.f1_score(Ym_test,Ym_test_pred, average = 'macro'))\nprint(\"f1 score micro :\",metrics.f1_score(Ym_test,Ym_test_pred, average = 'micro'))\nprint(\"Hamming loss :\",metrics.hamming_loss(Ym_test,Ym_test_pred))\nprint(\"Total Time taken = {}\".format(datetime.now() - start))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Logistic regression"},{"metadata":{"trusted":true},"cell_type":"code","source":" class LogisticRegression(object):\n    \n    def __init__(Logreg, alpha=0.01, n_iteration=100):   \n        Logreg.alpha = alpha                            \n        Logreg.n_iter = n_iteration\n        \n    def _sigmoid_function(Logreg, x): #This function is resonsible for calculating the sigmoid value with given parameter\n        value = 1 / (1 + np.exp(-x))\n        return value\n    def _cost_function(Logreg,h,theta, y): # The fuctions calculates the cost value\n        m = len(y)\n        cost = (1 / m) * (np.sum(-y.T.dot(np.log(h)) - (1 - y).T.dot(np.log(1 - h))))\n        return cost\n    \n    def _gradient_descent(Logreg,X,h,theta,y,m): # This function calculates the theta value by gradient descent\n        gradient_value = np.dot(X.T, (h - y)) / m\n        theta -= Logreg.alpha * gradient_value\n        return theta\n\n    def predict(Logreg, X): # this function calls the max predict function to classify the individul feauter\n        X = np.insert(X, 0, 1, axis=1)\n        X_predicted = [max((Logreg._sigmoid_function(i.dot(theta)), c) for theta, c in Logreg.theta)[1] for i in X ]\n        return X_predicted","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logi = LogisticRegression(n_iteration=30000).fit(X_train_multilabel, y_train)\ny_pred1 = logi.predict(Xtest_multilabel)\nprint_score(Ym_test, y_pred1)\nprint(\"Total Time taken = {}\".format(datetime.now() - start))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Multilabel KNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"knn = MLkNN(k=10)\nknn.fit(Xtrain, Ym_train)\ny_pred4 = knn.predict(Xtest)\nprint_score(Ym_test, y_pred4)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Unsupervised Learning Algorithms"},{"metadata":{},"cell_type":"markdown","source":"K-Means clustering"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = Xtrain\n# Divide into k groups using k-mean clustering\nmodel = KMeans(n_clusters=n_clusters, init='k-m\nkm = KMeans(n_clusters=1000, init='random', n_init=10, max_iter=300, tol=1e-04, random_state=0)\ny_km = km.fit_predict(X)\n\n# plot the centroids\nplt.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1],s=250, marker='*',c='red', edgecolor='black',label='centroids')\nplt.legend(scatterpoints=1)\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mean Shift"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.cluster import MeanShift\nms = MeanShift()\nms.fit(X_train_multilabel[:3000,].toarray())\ncluster_centers = ms.cluster_centers_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\nax.scatter(Xtrain[:,0], Xtrain[:,1], Xtrain[:,2], marker='o')\nax.scatter(cluster_centers[:,0], cluster_centers[:,1], cluster_centers[:,2], marker='x', color='red', s=300, linewidth=5, zorder=10)\nplt.show()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}