{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### **Tasks to accomplish**\n#### **1. Import the dataset effectively.**\n    1. Choose appropriate file format\n    2. Ensure chunks of data are created and ensure different cores are engaged\n#### **2. Perform EDA**\n    1. Understand the shape and statistics of data \n    2. Plot metrics of the data\n#### **3. Preprocessing**\n    1. Regex processsing and count words removal\n    2. Tokenize\n    3. Stemming\n    4. Vectorize and embeddings\n#### **4. Model Selection**\n#### **5. Model Deployment**","metadata":{}},{"cell_type":"code","source":"!unzip /kaggle/input/c/facebook-recruiting-iii-keyword-extraction/Train.zip\n!unzip /kaggle/input/c/facebook-recruiting-iii-keyword-extraction/Test.zip","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%config Completer.use_jedi = False","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:46:32.530767Z","iopub.execute_input":"2023-07-07T09:46:32.531676Z","iopub.status.idle":"2023-07-07T09:46:32.539905Z","shell.execute_reply.started":"2023-07-07T09:46:32.53164Z","shell.execute_reply":"2023-07-07T09:46:32.53863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport csv\nimport re\nimport datetime as dt\nimport joblib\n\nimport numpy as np\nimport pandas as pd\nimport dask.array as da\nimport dask.dataframe as dd\nfrom sqlalchemy import create_engine\nimport sqlite3\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n%matplotlib inline\n\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom skmultilearn.adapt import mlknn\n\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score,precision_score,recall_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-07T10:28:34.662568Z","iopub.execute_input":"2023-07-07T10:28:34.662996Z","iopub.status.idle":"2023-07-07T10:28:34.783985Z","shell.execute_reply.started":"2023-07-07T10:28:34.662964Z","shell.execute_reply":"2023-07-07T10:28:34.78309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = \"/kaggle/working/Train.csv\"\nTEST_PATH = \"/kaggle/working/Test.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Analysis","metadata":{}},{"cell_type":"code","source":"if not os.path.isfile('train.db'):\n    disk_engine = create_engine('sqlite:///train.db')\n    chunksize = 180000\n    j = 0\n    index_start = 1\n    for df in pd.read_csv(\n        TRAIN_PATH, \n        names=['Id', 'Title', 'Body', 'Tags'], chunksize=chunksize, \n        iterator=True, encoding='utf-8'\n    ):\n        df.index += index_start\n        j += 1\n        if(j%10 == 0):\n            print('{} rows'.format(j*chunksize))\n        df.to_sql('data', disk_engine, if_exists = 'append')\n        index_start = df.index[-1] + 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    con = sqlite3.connect('/kaggle/working/train.db')\n    df = pd.read_sql_query('SELECT COUNT(*) AS num_rows FROM data',con=con)\n    print(\"Number of rows : {}\".format(df['num_rows'].iloc[0]))\n    con.close()\nexcept FileNotFoundError:\n    print(\"Training database doesn't exist\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    con = sqlite3.connect('/kaggle/working/train.db')\n    df = pd.read_sql_query('SELECT Title,Body,Tags,COUNT(*) AS num_duplicates FROM data GROUP BY Title,Body,Tags',con=con)\n    con.close()\nexcept FileNotFoundError:\n    print(\"Training database doesn't exist\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:31:12.311401Z","iopub.execute_input":"2023-07-07T09:31:12.311763Z","iopub.status.idle":"2023-07-07T09:33:28.045193Z","shell.execute_reply.started":"2023-07-07T09:31:12.311735Z","shell.execute_reply":"2023-07-07T09:33:28.043493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:34:12.979053Z","iopub.execute_input":"2023-07-07T09:34:12.979449Z","iopub.status.idle":"2023-07-07T09:34:12.997756Z","shell.execute_reply.started":"2023-07-07T09:34:12.979418Z","shell.execute_reply":"2023-07-07T09:34:12.996904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Id'] = df.index + 1\ndf = df.set_index('Id')","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:52:22.850357Z","iopub.execute_input":"2023-07-07T09:52:22.85072Z","iopub.status.idle":"2023-07-07T09:52:23.382358Z","shell.execute_reply.started":"2023-07-07T09:52:22.850692Z","shell.execute_reply":"2023-07-07T09:52:23.381296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:52:30.243082Z","iopub.execute_input":"2023-07-07T09:52:30.24349Z","iopub.status.idle":"2023-07-07T09:52:30.256209Z","shell.execute_reply.started":"2023-07-07T09:52:30.243458Z","shell.execute_reply":"2023-07-07T09:52:30.255227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a new database with no duplicates\nif not os.path.isfile('train_no_dup.db'):\n    disk_dup = create_engine(\"sqlite:///train_no_dup.db\")\n    no_dup = pd.DataFrame(df, columns=['Id','Title', 'Body', 'Tags'])\n    no_dup.to_sql('no_dup_train', disk_dup)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T09:53:18.502231Z","iopub.execute_input":"2023-07-07T09:53:18.502703Z","iopub.status.idle":"2023-07-07T09:53:18.509598Z","shell.execute_reply.started":"2023-07-07T09:53:18.502664Z","shell.execute_reply":"2023-07-07T09:53:18.508732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Analysis of tags","metadata":{}},{"cell_type":"code","source":"tags = pd.DataFrame(df.Tags)\ntags.fillna('',inplace=True)\nvectorizer = CountVectorizer(tokenizer = lambda x: x.split())\ntag_dtm = vectorizer.fit_transform(tags['Tags'])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:00:48.487598Z","iopub.execute_input":"2023-07-07T10:00:48.488019Z","iopub.status.idle":"2023-07-07T10:01:15.083496Z","shell.execute_reply.started":"2023-07-07T10:00:48.487989Z","shell.execute_reply":"2023-07-07T10:01:15.082509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tag_names = vectorizer.get_feature_names_out()\nprint(\"Number of data points :\", tags.shape[0])\nprint(\"Number of unique tags :\", tag_dtm.shape[1])\nprint(\"Some of the tags we have :\", tag_names[:10])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:04:52.145863Z","iopub.execute_input":"2023-07-07T10:04:52.146269Z","iopub.status.idle":"2023-07-07T10:04:52.187423Z","shell.execute_reply.started":"2023-07-07T10:04:52.146211Z","shell.execute_reply":"2023-07-07T10:04:52.186452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('stackoverflow.csv', index_col=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:09:26.605664Z","iopub.execute_input":"2023-07-07T10:09:26.606613Z","iopub.status.idle":"2023-07-07T10:09:29.089883Z","shell.execute_reply.started":"2023-07-07T10:09:26.606576Z","shell.execute_reply":"2023-07-07T10:09:29.08886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\ndf['Tags'] = df['Tags'].apply(lambda x: ast.literal_eval(x))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:11:17.511479Z","iopub.execute_input":"2023-07-07T10:11:17.511931Z","iopub.status.idle":"2023-07-07T10:11:18.307451Z","shell.execute_reply.started":"2023-07-07T10:11:17.5119Z","shell.execute_reply":"2023-07-07T10:11:18.306482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Target Variable Encoding\nmultilabel = MultiLabelBinarizer()\ny = multilabel.fit_transform(df['Tags'])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:12:25.543174Z","iopub.execute_input":"2023-07-07T10:12:25.544197Z","iopub.status.idle":"2023-07-07T10:12:25.624217Z","shell.execute_reply.started":"2023-07-07T10:12:25.54416Z","shell.execute_reply":"2023-07-07T10:12:25.623229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multilabel.classes_","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:12:51.070818Z","iopub.execute_input":"2023-07-07T10:12:51.071225Z","iopub.status.idle":"2023-07-07T10:12:51.07762Z","shell.execute_reply.started":"2023-07-07T10:12:51.071193Z","shell.execute_reply":"2023-07-07T10:12:51.076721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(y, columns=multilabel.classes_)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:13:09.133078Z","iopub.execute_input":"2023-07-07T10:13:09.133794Z","iopub.status.idle":"2023-07-07T10:13:09.156161Z","shell.execute_reply.started":"2023-07-07T10:13:09.133759Z","shell.execute_reply":"2023-07-07T10:13:09.15513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf = TfidfVectorizer(analyzer='word', max_features=10000, ngram_range=(1,3), stop_words='english')\nX = tfidf.fit_transform(df['Text'])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:14:00.672964Z","iopub.execute_input":"2023-07-07T10:14:00.673367Z","iopub.status.idle":"2023-07-07T10:14:44.174836Z","shell.execute_reply.started":"2023-07-07T10:14:00.67333Z","shell.execute_reply":"2023-07-07T10:14:44.173796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:16:10.800809Z","iopub.execute_input":"2023-07-07T10:16:10.801312Z","iopub.status.idle":"2023-07-07T10:16:10.836969Z","shell.execute_reply.started":"2023-07-07T10:16:10.801225Z","shell.execute_reply":"2023-07-07T10:16:10.835959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_metrics(y_test,predictions):\n    jaccard = np.minimum(y_test,predictions).sum(axis = 1)/np.maximum(y_test,predictions).sum(axis = 1) \n    print(\"accuracy :\",metrics.accuracy_score(y_test,predictions))\n    print(\"macro f1 score :\",metrics.f1_score(y_test, predictions, average = 'macro'))\n    print(\"micro f1 score :\",metrics.f1_score(y_test, predictions, average = 'micro'))\n    print(\"hamming loss :\",metrics.hamming_loss(y_test,predictions))\n    print(\"Jaccard score :\",jaccard.mean()*100)\n    print(\"Precision recall report :\\n\",metrics.classification_report(y_test, predictions))\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:25:56.779951Z","iopub.execute_input":"2023-07-07T10:25:56.780375Z","iopub.status.idle":"2023-07-07T10:25:56.788346Z","shell.execute_reply.started":"2023-07-07T10:25:56.780335Z","shell.execute_reply":"2023-07-07T10:25:56.787268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#OVR models\nsgd = SGDClassifier(loss='log', alpha=0.00001, penalty='l1')\nclassifier = OneVsRestClassifier(sgd, n_jobs=-1)\nclassifier.fit(X_train, y_train)\npredictions = classifier.predict(X_test)\nprint_metrics(y_test,predictions)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:27:05.31333Z","iopub.execute_input":"2023-07-07T10:27:05.313723Z","iopub.status.idle":"2023-07-07T10:27:09.249297Z","shell.execute_reply.started":"2023-07-07T10:27:05.313695Z","shell.execute_reply":"2023-07-07T10:27:09.248369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svm = LinearSVC() \nclassifier = OneVsRestClassifier(svm, n_jobs=-1)\nclassifier.fit(X_train, y_train)\npredictions = classifier.predict(X_test)\nprint_metrics(y_test,predictions)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:28:47.126769Z","iopub.execute_input":"2023-07-07T10:28:47.127149Z","iopub.status.idle":"2023-07-07T10:28:49.992442Z","shell.execute_reply.started":"2023-07-07T10:28:47.127119Z","shell.execute_reply":"2023-07-07T10:28:49.991414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(solver='lbfgs')\nclassifier = OneVsRestClassifier(lr, n_jobs=-1)\nclassifier.fit(X_train, y_train)\npredictions = classifier.predict(X_test)\nprint_metrics(y_test,predictions)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:29:47.820166Z","iopub.execute_input":"2023-07-07T10:29:47.821105Z","iopub.status.idle":"2023-07-07T10:29:58.167571Z","shell.execute_reply.started":"2023-07-07T10:29:47.821068Z","shell.execute_reply":"2023-07-07T10:29:58.16653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}