{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import linear_model\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import classification_report, accuracy_score, recall_score, precision_score, f1_score\nfrom sklearn.model_selection import KFold\nimport numpy as np\nfrom nltk.tokenize import word_tokenize\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer, TfidfVectorizer\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\nimport nltk\nnltk.download('stopwords')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_set = \"../input/bigdata2021classification/train.csv/train.csv\"\ntest_set = \"../input/bigdata2021classification/test_without_labels.csv/test_without_labels.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=pd.read_csv(train_set,header=0)\n\nX = df_train.Content\ny = df_train.Label\n\nbow_svm = Pipeline([\n    (\"vectorizer\",CountVectorizer(stop_words=stopwords.words('english'))),\n\t(\"tfidf\",TfidfTransformer()), \n\t(\"SVM\",LinearSVC(max_iter=10000))\n\t])\nbow_rf = Pipeline([\n    (\"vectorizer\",CountVectorizer(stop_words=stopwords.words('english'))),\n\t(\"tfidf\",TfidfTransformer()), \n\t(\"Random Forest\",RandomForestClassifier(n_estimators=100))\n\t])\nsvd_svm = Pipeline([\n    (\"vectorizer\",CountVectorizer(stop_words=stopwords.words('english'))),\n\t(\"tfidf\",TfidfTransformer()),\n\t('feat', TruncatedSVD(n_components = 600)),\n    (\"SVM\",LinearSVC(max_iter=10000))\n\t])\nsvd_rf = Pipeline([\n    (\"vectorizer\",CountVectorizer(stop_words=stopwords.words('english'))),\n\t(\"tfidf\",TfidfTransformer()),\n\t('feat', TruncatedSVD(n_components = 600)),\n\t(\"Random Forest\",RandomForestClassifier(n_estimators=100))\n\t])\nmy_method = Pipeline([\n    (\"vectorizer\",CountVectorizer(stop_words=stopwords.words('english'))),\n\t(\"tfidf\",TfidfTransformer()),\n    (\"KNN\",KNeighborsClassifier(n_neighbors=3))\n    ])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models=[\n    (\"SVM (BoW)\", bow_svm),\n    (\"Random Forest (BoW)\", bow_rf),\n    (\"SVM (SVD)\", svd_svm),\n    (\"Random Forest (SVD)\", svd_rf),\n    (\"My Method\", my_method)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = {'Statistic Measure': ['Accuracy', 'Precision', 'Recall', 'F-Measure']}\nevaluation = pd.DataFrame(data=d).set_index('Statistic Measure')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = KFold(n_splits=5)\nfor name, model in models:\n\ts = pd.Series()\n\tfor train, test in kf.split(X):\n\t\tX_train, X_test = X[train], X[test]\n\t\ty_train, y_test = y[train], y[test]\n\t\ttrain = model.fit(X_train, y_train)\n\t\typred = model.predict(X_test)\n\td = [accuracy_score(y_test, ypred), precision_score(y_test, ypred, average = 'macro'), recall_score(y_test, ypred, average = 'macro'), f1_score(y_test, ypred, average = 'macro')]\n\tevaluation[name] = d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluation.to_csv('/kaggle/working/EvaluationMetric_5fold.csv', sep = ',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = pd.read_csv(test_set, sep = ',')\ncont = test_data.Content\nid_n = test_data.Id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = bow_svm.fit(X, y)\npredictions = bow_svm.predict(cont)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_table = pd.DataFrame(data = id_n, columns = ['Id'])\nprediction_table['Predicted'] = predictions\nprediction_table.to_csv('/kaggle/working/testSet_categories.csv', sep =',', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}