{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install kaggle","metadata":{"id":"utBVKTjVqtOa","outputId":"a78fb623-11fe-4405-dcce-dddb949235e3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from google.colab import files\n  \n  \nuploaded = files.upload()\n","metadata":{"id":"PMGM617NrYNN","outputId":"b709c848-eaf7-45eb-b408-c9aa15c2561d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir ~/.kaggle","metadata":{"id":"ik1xvGf3rkLC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! cp kaggle.json ~/.kaggle/","metadata":{"id":"gutoeAWjrxJK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! chmod 600 ~/.kaggle/kaggle.json","metadata":{"id":"m58vyMEPr4WL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle competitions download -c facebook-recruiting-iii-keyword-extraction","metadata":{"id":"8M7LGkvBr8Yv","outputId":"e69c7f58-e5a7-4185-a0b8-61bf7f28b88d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install scikit-multilearn","metadata":{"id":"VjmH1M-nsbsF","outputId":"f6d393d0-4dfc-48e7-b087-f6510410e70c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas as pd\nimport sqlite3\nimport csv\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom wordcloud import WordCloud\nimport re\nimport os\nfrom sqlalchemy import create_engine # database connection\nimport datetime as dt\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.snowball import SnowballStemmer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.model_selection import GridSearchCV,RandomizedSearchCV\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score,precision_score,recall_score\nfrom sklearn import svm\nfrom sklearn.linear_model import LogisticRegression\nfrom skmultilearn.adapt import mlknn\nfrom skmultilearn.problem_transform import ClassifierChain\nfrom skmultilearn.problem_transform import BinaryRelevance\nfrom skmultilearn.problem_transform import LabelPowerset\nfrom sklearn.naive_bayes import GaussianNB\nfrom datetime import datetime","metadata":{"id":"gxhMqHRauOjW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! unzip Train.zip","metadata":{"id":"Likmi34ouTKK","outputId":"172235ae-9243-4b56-d67a-5ad0328971b7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile('train.db'):\n    start = datetime.now()\n    disk_engine = create_engine('sqlite:///train.db')\n    start = dt.datetime.now()\n    chunksize = 180000\n    j = 0\n    index_start = 1\n    for df in pd.read_csv(\n        '/content/Train.csv', \n        names=['Id', 'Title', 'Body', 'Tags'], chunksize=chunksize, \n        iterator=True, encoding='utf-8'\n    ):\n        df.index += index_start\n        j += 1\n        print('{} rows'.format(j*chunksize))\n        df.to_sql('data', disk_engine, if_exists = 'append')\n        index_start = df.index[-1] + 1\n    print(\"Time taken to run this cell: \", datetime.now() - start)","metadata":{"id":"LFv_chq-ulXG","outputId":"ad26e7b8-45cc-4154-e730-ab3b04728957"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile('train.db'):\n    start = datetime.now()\n    con = sqlite3.connect('train.db')\n    num_rows = pd.read_sql_query(\"\"\"SELECT count(*) FROM data\"\"\", con)\n    print(\"Number of rows in the database: \",\"\\n\",num_rows['count(*)'].values[0])\n    con.close()  # Always remember to close the database\n    print(\"Time taken to count the number of rows: \", datetime.now() - start)\nelse:\n    print(\"Please download the train.db file from drive or run the above cell to genarate train.db file\")","metadata":{"id":"cPEGeEsHvJ6N","outputId":"5dcab174-c62e-4f19-b110-2556c872d7e9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile('train.db'):\n    start = datetime.now()\n    con = sqlite3.connect('train.db')\n    df_no_dup = pd.read_sql_query('SELECT Title, Body, Tags, COUNT(*) as cnt_dup FROM data GROUP BY Title, Body, Tags', con)\n    con.close()\n    print(\"Time taken to run this cell: \", datetime.now() - start)\nelse:\n    print(\"Please download the train.db file from drive or run the first to genarate train.db file\")","metadata":{"id":"d-sYOUKQxtqq","outputId":"55711d2b-d7bb-4ed2-b7f9-21932bfbb0f1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_no_dup.head()","metadata":{"id":"m1IrX3Isx15q","outputId":"107bec16-cb80-46ea-afc1-ea8855796eca"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.sum(df_no_dup['cnt_dup']))        # Total Number of Questions\nprint(num_rows['count(*)'].values[0])      # Total Number of Questions\nprint(df_no_dup.shape[0])                  # Total Number of Unique Questions","metadata":{"id":"3q4qsZGJy-eK","outputId":"2503e35f-2b3c-423b-b404-6075a3740dd5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of duplicate questions: \", num_rows['count(*)'].values[0] - df_no_dup.shape[0],\n  \"(\",(1 - ((df_no_dup.shape[0])/(num_rows['count(*)'].values[0])))*100,\"% )\")","metadata":{"id":"ZymcTMyPzDhI","outputId":"f6810aab-4d46-4c7f-8dda-7c8ac9cbde96"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_no_dup.cnt_dup.value_counts()","metadata":{"id":"uBX7S3FNzJya","outputId":"404b3e52-ddd8-4f31-d32b-78d7d068c660"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_no_dup.info(show_counts = True)","metadata":{"id":"2FM2qUsXzO0X","outputId":"e9e98a0e-f03d-4f5f-9dcf-a58d6934fbe7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.now()\ndf_no_dup[\"tag_count\"] = df_no_dup[\"Tags\"].apply(\n    lambda text: len(text.split(\" \")) if text != None else 0 \n)\n\n# Adding a new feature number of tags per question\nprint(\"Time taken to run this cell: \", datetime.now() - start)\ndf_no_dup.head()","metadata":{"id":"yrX9icUbzTx-","outputId":"98c6a061-651b-451a-e31b-84969aa45e57"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_no_dup.tag_count.value_counts()","metadata":{"id":"h6KDv-SVzevW","outputId":"9bb9e729-6850-4d8c-a719-a121291c46a4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile('content/train_no_dup.db'):\n    disk_dup = create_engine(\"sqlite:///train_no_dup.db\")\n    no_dup = pd.DataFrame(df_no_dup, columns=['Title', 'Body', 'Tags'])\n    no_dup.to_sql('no_dup_train', disk_dup)","metadata":{"id":"sNYl9glnzk8E"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile('/content/train_no_dup.db'):\n    start = datetime.now()\n    con = sqlite3.connect('/content/train_no_dup.db')\n    tag_data = pd.read_sql_query(\"\"\"SELECT Tags FROM no_dup_train\"\"\", con)\n    # Always remember to close the database\n    con.close()\n\n    # Let's now drop unwanted column.\n    tag_data.dropna()\n    # Printing first 5 rows from our data frame\n    tag_data.head()\n","metadata":{"id":"XfRI8jxA0YI-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_data.dropna(axis=0, how='any',inplace=True)\n","metadata":{"id":"bP4rxaUyGhGO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(tokenizer = lambda x: x.split())","metadata":{"id":"pO6qjwB21fAE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_dtm = vectorizer.fit_transform(tag_data['Tags'])","metadata":{"id":"Oo1H6Ohh2WL2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_dtm = vectorizer.fit_transform(tag_data['Tags'])","metadata":{"id":"TpTNR_8pCSY-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of data points :\", tag_dtm.shape[0])\nprint(\"Number of unique tags :\", tag_dtm.shape[1])","metadata":{"id":"D_yBWVbgDHFC","outputId":"cd90d6f4-bbe1-4059-8c60-be12868c8fe4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'get_feature_name()' gives us the vocabulary.\ntags = vectorizer.get_feature_names()\n# Let's look at the tags we have.\nprint(\"Some of the tags we have :\", tags[:10])\n","metadata":{"id":"V9YALDCwDOHe","outputId":"c117d81f-c333-472e-e6fa-942828dde6f2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"freqs = tag_dtm.sum(axis = 0).A1\nresult = dict(zip(tags, freqs))","metadata":{"id":"xHA6-yfmS_Wu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile('/content/tag_counts_dict_dtm.csv'):\n    with open('tag_counts_dict_dtm.csv', 'w') as csv_file:\n        writer = csv.writer(csv_file)\n        for key, value in result.items():\n            writer.writerow([key, value])\ntag_df = pd.read_csv(\"/content/tag_counts_dict_dtm.csv\", names=['Tags', 'Counts'])\ntag_df.head()","metadata":{"id":"BKFDluAjTDzs","outputId":"25e987ef-fd6c-49a0-d962-8b183e97802b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_df_sorted = tag_df.sort_values(['Counts'], ascending=False)\ntag_counts = tag_df_sorted['Counts'].values","metadata":{"id":"NB_JVFnaTYdT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_counts","metadata":{"id":"WF-1VUZMTbaQ","outputId":"a3453bc2-78d5-4673-e292-89a9cc90c44b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(tag_counts)\nplt.title(\"Distribution of number of times tag appeared in questions\")\nplt.grid()\nplt.xlabel(\"Tag number\")\nplt.ylabel(\"Number of times tag appeared\")\nplt.show()","metadata":{"id":"amCdErPrUUAa","outputId":"06825571-5d25-4325-b50a-583874b14d3b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(tag_counts[0:100], c='b')\nplt.scatter(x=list(range(0,100,5)), y=tag_counts[0:100:5], c='orange', label=\"quantiles with 0.05 intervals\")\n# Quantiles with 0.25 difference\nplt.scatter(x=list(range(0,100,25)), y=tag_counts[0:100:25], c='m', label = \"quantiles with 0.25 intervals\")\nfor x,y in zip(list(range(0,100,25)), tag_counts[0:100:25]):\n    plt.annotate(s=\"({} , {})\".format(x,y), xy=(x,y), xytext=(x-0.05, y+500))\n\nplt.title('First 100 tags: Distribution of number of times tag appeared questions')\nplt.grid()\nplt.xlabel(\"Tag number\")\nplt.ylabel(\"Number of times tag appeared\")\nplt.legend()\nplt.show()\nprint(len(tag_counts[0:100:5]), tag_counts[0:100:5])","metadata":{"id":"M26vzH5FUec3","outputId":"1ab58af0-015a-49f0-df06-0e132e6fac44"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lst_tags_gt_10k = tag_df[tag_df.Counts>10000].Tags\n#Print the length of the list\nprint ('{} Tags are used more than 10000 times'.format(len(lst_tags_gt_10k)))\n# Store tags greater than 100K in one list\nlst_tags_gt_100k = tag_df[tag_df.Counts>100000].Tags\n#Print the length of the list.\nprint ('{} Tags are used more than 100000 times'.format(len(lst_tags_gt_100k)))","metadata":{"id":"Buuuo_BSU30L","outputId":"ef9d27b4-86d8-4ba9-8646-8393a4e4c386"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_quest_count = tag_dtm.sum(axis=1).tolist()\n# Converting list of lists into single list, we will get [[3], [4], [2], [2], [3]] and we are converting this to [3, 4, 2, 2, 3]\ntag_quest_count=[int(j) for i in tag_quest_count for j in i]\nprint ('We have total {} datapoints.'.format(len(tag_quest_count)))\n\nprint(tag_quest_count[:5])","metadata":{"id":"tkcHarEjViwd","outputId":"a3f63e7f-09a3-4730-d681-d290221da689"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( \"Maximum number of tags per question: %d\"%max(tag_quest_count))\nprint( \"Minimum number of tags per question: %d\"%min(tag_quest_count))\nprint( \"Avg. number of tags per question: %f\"% ((sum(tag_quest_count)*1.0)/len(tag_quest_count)))","metadata":{"id":"FACNjUJmVpPf","outputId":"60e086db-967f-4582-f3af-12c86337861a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(tag_quest_count, palette='gist_rainbow')\nplt.title(\"Number of tags in the questions \")\nplt.xlabel(\"Number of Tags\")\nplt.ylabel(\"Number of questions\")\nplt.show()","metadata":{"id":"hxAAgDg9Vx1z","outputId":"f8eb373d-7eb3-4d87-c8de-332db82b525b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting word cloud\nstart = datetime.now()\n\n# Lets first convert the 'result' dictionary to 'list of tuples'\ntup = dict(result.items())\n# Initializing WordCloud using frequencies of tags.\nwordcloud = WordCloud(    \n    background_color='black', width=1600, height=800,\n).generate_from_frequencies(tup)\n\nfig = plt.figure(figsize=(30,20))\nplt.imshow(wordcloud)\nplt.axis('off')\nplt.tight_layout(pad=0)\nfig.savefig(\"tag.png\")\nplt.show()\nprint(\"Time taken to run this cell :\", datetime.now() - start)","metadata":{"id":"1TyHOkjuV0W0","outputId":"ec1e03d8-0d03-432e-a171-3c7f7a7eb2cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = np.arange(20)\ntag_df_sorted.head(20).plot(kind='bar')\nplt.title('Frequency of top 20 tags')\nplt.xticks(i, tag_df_sorted.head(20)[\"Tags\"])\nplt.xlabel('Tags')\nplt.ylabel('Counts')\nplt.show()","metadata":{"id":"fdQIt090V8Wo","outputId":"cc99f323-1723-4304-8ce2-c61ba95f7aff"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" import nltk\n nltk.download('stopwords')\ndef striphtml(data):\n    cleanr = re.compile('<.*?>')\n    cleantext = re.sub(cleanr, ' ', str(data))\n    return cleantext\nstop_words = set(stopwords.words('english'))\nstemmer = SnowballStemmer(\"english\")","metadata":{"id":"O95NFDS8WAve","outputId":"614439f7-5a4c-4634-aa39-3bd6fa26dfbc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_connection(db_file):\n    \"\"\" \n    Create a database connection to the SQLite database specified by db_file\n    :param db_file: database file\n    :return: Connection object or None\n    \"\"\"\n    try:\n        conn = sqlite3.connect(db_file)\n        return conn\n    except Error as e:\n        print(e)\n    return None\n\ndef create_table(conn, create_table_sql):\n    \"\"\" \n    Create a table from the create_table_sql statement\n    :param conn: Connection object\n    :param create_table_sql: a CREATE TABLE statement\n    :return:\n    \"\"\"\n    try:\n        c = conn.cursor()\n        c.execute(create_table_sql)\n    except Error as e:\n        print(e)\n        \ndef checkTableExists(dbcon):\n    cursr = dbcon.cursor()\n    str = \"select name from sqlite_master where type='table'\"\n    table_names = cursr.execute(str)\n    print(\"Tables in the databse:\")\n    tables = table_names.fetchall() \n    print(tables[0][0])\n    return(len(tables))\n\ndef create_database_table(database, query):\n    conn = create_connection(database)\n    if conn is not None:\n        create_table(conn, query)\n        checkTableExists(conn)\n    else:\n        print(\"Error! cannot create the database connection.\")\n    conn.close()\n\nsql_create_table = \"\"\"CREATE TABLE IF NOT EXISTS QuestionsProcessed (question text NOT NULL, code text, tags text, words_pre integer, words_post integer, is_code integer);\"\"\"\ncreate_database_table(\"Processed.db\", sql_create_table)","metadata":{"id":"qcmrTIr7WUkz","outputId":"50667fe6-395e-47b4-d743-ac46159231df"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.now()\nread_db = '/content/train_no_dup.db'\nwrite_db = 'Processed.db'\nif os.path.isfile(read_db):\n    conn_r = create_connection(read_db)\n    if conn_r is not None:\n        reader = conn_r.cursor()\n        reader.execute(\"SELECT Title, Body, Tags From no_dup_train ORDER BY RANDOM() LIMIT 100000;\")\nif os.path.isfile(write_db):\n    conn_w = create_connection(write_db)\n    if conn_w is not None:\n        tables = checkTableExists(conn_w)\n        writer = conn_w.cursor()\n        if tables != 0:\n            writer.execute(\"DELETE FROM QuestionsProcessed WHERE 1\")\n            print(\"Cleared all the rows\")\n\nprint(\"Time taken to run this cell: \", datetime.now() - start)\n","metadata":{"id":"pD8dXeUoXeg3","outputId":"bf3d04c0-e815-4568-be75-b4b4b42add1c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')\nstart = datetime.now()\npreprocessed_data_list = []\nreader.fetchone()\nquestions_with_code=0\nlen_pre=0\nlen_post=0\nquestions_processed = 0\nfor row in reader:\n    is_code = 0\n    title, question, tags = row[0], row[1], row[2]\n    if '<code>' in question:\n        questions_with_code += 1\n        is_code = 1\n    x = len(question) + len(title)\n    len_pre += x\n\n    code = str(re.findall(r'<code>(.*?)</code>', question, flags=re.DOTALL))\n    question = re.sub('<code>(.*?)</code>', '', question, flags=re.MULTILINE|re.DOTALL)\n    question = striphtml(question.encode('utf-8'))\n    title = title.encode('utf-8')\n    question = str(title) + \" \" + str(question)\n    question = re.sub(r'[^A-Za-z]+',' ',question)\n    words = word_tokenize(str(question.lower()))\n    # Removing all single letter and stopwords from question except for the letter 'c'\n    question=' '.join(\n        str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j) != 1 or j == 'c')\n    )\n\n    len_post += len(question)\n    tup = (question, code, tags, x, len(question), is_code)\n    questions_processed += 1\n    writer.execute(\"insert into QuestionsProcessed(question,code,tags,words_pre,words_post,is_code) values (?,?,?,?,?,?)\",tup)\n    if (questions_processed % 10000 == 0):\n        print(\"Number of questions completed = \",questions_processed)\n\nno_dup_avg_len_pre = (len_pre*1.0) / questions_processed\nno_dup_avg_len_post = (len_post*1.0) / questions_processed\n\nprint(\"Avg length of questions(Title+Body) before processing: %d\"%no_dup_avg_len_pre)\nprint(\"Avg length of questions(Title+Body) after processing: %d\"%no_dup_avg_len_post)\nprint(\"Percent of questions containing code: %d\"%((questions_with_code*100.0)/questions_processed))\nprint(\"Time taken to run this cell :\", datetime.now() - start)","metadata":{"id":"EDJXiOzXX537","outputId":"83b827f8-7acb-4bf2-856a-1c87334880a9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Don't forget to close the connections, or else you will end up with locks\nconn_r.commit()\nconn_w.commit()\nconn_r.close()\nconn_w.close()","metadata":{"id":"A098yOKqaaYG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile(write_db):\n    conn_r = create_connection(write_db)\n    if conn_r is not None:\n        reader =conn_r.cursor()\n        reader.execute(\"SELECT question From QuestionsProcessed LIMIT 10\")\n        print(\"Questions after preprocessed\")\n        print('='*100)\n        reader.fetchone()\n        for row in reader:\n            print(row)\n            print('-'*100)\nconn_r.commit()\nconn_r.close()","metadata":{"id":"inSaiw85bRRh","outputId":"4d988723-0bd3-4f96-ec73-59d82cbbabff"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_db = 'Processed.db'\nif os.path.isfile(write_db):\n    conn_r = create_connection(write_db)\n    if conn_r is not None:\n        preprocessed_data = pd.read_sql_query(\"\"\"SELECT question, Tags FROM QuestionsProcessed\"\"\", conn_r)\nconn_r.commit()\nconn_r.close()","metadata":{"id":"lK7Xdi2jbV_3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_data.head()","metadata":{"id":"VdzgIgeFbfgo","outputId":"3e95751c-1a34-485e-b1d0-c800a36eb3cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of data points in sample: \", preprocessed_data.shape[0])\nprint(\"Number of dimensions: \", preprocessed_data.shape[1])","metadata":{"id":"u_tOAt_NbjY7","outputId":"fd6e7a44-46da-4a01-d1d7-854e86ff91be"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# binary='true' will give a binary vectorizer\nvectorizer = CountVectorizer(tokenizer = lambda x: x.split(), binary='true')\nmultilabel_y = vectorizer.fit_transform(preprocessed_data['tags'])","metadata":{"id":"lXj-9J2gbmn4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tags_to_choose(n):\n    t = multilabel_y.sum(axis=0).tolist()[0]\n    sorted_tags_i = sorted(range(len(t)), key=lambda i: t[i], reverse=True)\n    multilabel_yn = multilabel_y[:,sorted_tags_i[:n]]\n    return multilabel_yn\n\ndef questions_explained_fn(n):\n    multilabel_yn = tags_to_choose(n)\n    x = multilabel_yn.sum(axis=1)\n    return (np.count_nonzero(x==0))","metadata":{"id":"qmDJcDjPbq_O"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting 500 Tags\nquestions_explained = []\ntotal_tags = multilabel_y.shape[1]\ntotal_qs = preprocessed_data.shape[0]\nfor i in range(500, total_tags, 100):\n    questions_explained.append(np.round(((total_qs-questions_explained_fn(i))/total_qs)*100,3))","metadata":{"id":"9nmUKw6lbuLn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(questions_explained)\nxlabel = list(500+np.array(range(-50,450,50))*50)\nax.set_xticklabels(xlabel)\nplt.xlabel(\"Number of tags\")\nplt.ylabel(\"Number Questions coverd partially\")\nplt.grid()\nplt.show()\n# you can choose any number of tags based on your computing power, minimun is 500(it covers 90% of the tags)\nprint(\"With \",5500,\"tags we are covering \",questions_explained[50],\"% of questions\")\nprint(\"With \",500,\"tags we are covering \",questions_explained[0],\"% of questions\")","metadata":{"id":"Mk3L1Z0hcRUf","outputId":"e11b5918-d310-4623-d32f-76e4d9a22c6d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will be taking 500 tags\nmultilabel_yx = tags_to_choose(500)\nprint(\"Number of questions that are not covered :\", questions_explained_fn(500),\"out of \", total_qs)","metadata":{"id":"Vs15f6ICcYQk","outputId":"6e012638-9537-4271-e02e-52758bb81009"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_size = preprocessed_data.shape[0]\ntrain_size = int(0.80*total_size)\n\nx_train = preprocessed_data.head(train_size)\nx_test = preprocessed_data.tail(total_size - train_size)\n\ny_train = multilabel_yx[0:train_size, : ]\ny_test = multilabel_yx[train_size:total_size, : ]","metadata":{"id":"74504nC3ccY6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of data points in train data:\", y_train.shape)\nprint(\"Number of data points in test data:\", y_test.shape)","metadata":{"id":"0UEgc3psdZLe","outputId":"f659dc9d-7631-43a8-9be9-dc2b2a8dba99"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"./train.db\")","metadata":{"id":"bt46kPGadbrh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.now()\nvectorizer = TfidfVectorizer(\n    min_df = 0.00009, max_features = 10000, smooth_idf = True, norm = \"l2\",\n    tokenizer = lambda x: x.split(), sublinear_tf = False, ngram_range = (1,2)\n)\nx_train_multilabel = vectorizer.fit_transform(x_train['question'])\nx_test_multilabel = vectorizer.transform(x_test['question'])\nprint(\"Time taken to run this cell :\", datetime.now() - start)","metadata":{"id":"affHO3rPdiMX","outputId":"f6e419ec-1669-4f1e-9511-b5265246f6b6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Dimensions of train data X:\", x_train_multilabel.shape, \"Y :\",y_train.shape)\nprint(\"Dimensions of test data X:\", x_test_multilabel.shape, \"Y:\",y_test.shape)","metadata":{"id":"EF8L7Ccldvio","outputId":"a8d75a2c-0606-4269-9844-7aa537167f2d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = OneVsRestClassifier(SGDClassifier(loss='log', alpha=0.00001, penalty='l1'), n_jobs=-1)\nclassifier.fit(x_train_multilabel, y_train)\npredictions = classifier.predict(x_test_multilabel)","metadata":{"id":"yFISzIM9d3ma"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!pip install joblib --upgrade","metadata":{"id":"F3lJ8jAyeJ7A","outputId":"17b59ed8-8af3-4ff2-ed66-108dc137b424"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\njoblib.dump(classifier, 'lr.pkl') ","metadata":{"id":"yohCOvmBf1uX","outputId":"37b9ca64-30b3-4a4f-9168-80737018239a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier1 = joblib.load('/content/lr.pkl') ","metadata":{"id":"EMo9k9JtiazE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = classifier.predict(x_test_multilabel)","metadata":{"id":"gjApmPMwjS4N"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Accuracy :\",metrics.accuracy_score(y_test, predictions))\nprint(\"Hamming loss \",metrics.hamming_loss(y_test,predictions))\n\nprecision = precision_score(y_test, predictions, average='micro')\nrecall = recall_score(y_test, predictions, average='micro')\nf1 = f1_score(y_test, predictions, average='micro')","metadata":{"id":"gZzO-uwJjYV6","outputId":"360f53fe-38ca-46a1-9416-c5ee71dc861d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Micro-average quality numbers\")\nprint(\"Precision: {:.4f}, Recall: {:.4f}, F1-measure: {:.4f}\".format(precision, recall, f1))\n\nprecision = precision_score(y_test, predictions, average='macro')\nrecall = recall_score(y_test, predictions, average='macro')\nf1 = f1_score(y_test, predictions, average='macro')\n \nprint(\"Macro-average quality numbers\")\nprint(\"Precision: {:.4f}, Recall: {:.4f}, F1-measure: {:.4f}\".format(precision, recall, f1))","metadata":{"id":"QwNjpfRIjnwS","outputId":"f6a21562-6ffb-4a1c-f2b3-87a516a63bf6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (metrics.classification_report(y_test, predictions))","metadata":{"id":"5opyNjNyjwyF","outputId":"1ee4dae5-ace4-48fb-98b8-060867cd863f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Sy-sl02gkBl3"},"execution_count":null,"outputs":[]}]}