{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-05T17:38:15.899288Z","iopub.execute_input":"2023-04-05T17:38:15.899702Z","iopub.status.idle":"2023-04-05T17:38:15.910214Z","shell.execute_reply.started":"2023-04-05T17:38:15.899668Z","shell.execute_reply":"2023-04-05T17:38:15.908971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Analysis\nimport pandas as pd\nimport numpy as np\nfrom numpy import asarray\nfrom numpy import savetxt\nfrom numpy import loadtxt\nimport pickle as pkl\nfrom scipy import sparse\n\n# Data Visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport wordcloud\nfrom wordcloud import WordCloud, STOPWORDS\n\n# Text Processing\nimport re\nimport itertools\nimport string\nimport collections\nfrom collections import Counter\nfrom sklearn.preprocessing import LabelEncoder\nimport nltk\nfrom nltk.classify import NaiveBayesClassifier\nfrom nltk.corpus import stopwords\nfrom nltk import word_tokenize\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import PorterStemmer, WordNetLemmatizer\n\n# Machine Learning packages\nimport sklearn\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\nimport sklearn.cluster as cluster\nfrom sklearn.manifold import TSNE\n\n# Model training and evaluation\nfrom sklearn.model_selection import train_test_split\n\n#Models\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom xgboost import XGBClassifier\nfrom xgboost import plot_importance\n\n#Metrics\nfrom sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, accuracy_score, balanced_accuracy_score\nfrom sklearn.metrics import precision_score, recall_score, f1_score, multilabel_confusion_matrix, confusion_matrix\nfrom sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:15.912464Z","iopub.execute_input":"2023-04-05T17:38:15.912837Z","iopub.status.idle":"2023-04-05T17:38:15.924069Z","shell.execute_reply.started":"2023-04-05T17:38:15.912801Z","shell.execute_reply":"2023-04-05T17:38:15.92313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## get the clean data for train\n#train = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/clean_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:15.925253Z","iopub.execute_input":"2023-04-05T17:38:15.926197Z","iopub.status.idle":"2023-04-05T17:38:15.937283Z","shell.execute_reply.started":"2023-04-05T17:38:15.926148Z","shell.execute_reply":"2023-04-05T17:38:15.936108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading dataset\ndata_set = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/raw_train.csv\")\ndata_set.tail()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:15.940141Z","iopub.execute_input":"2023-04-05T17:38:15.940885Z","iopub.status.idle":"2023-04-05T17:38:16.491043Z","shell.execute_reply.started":"2023-04-05T17:38:15.940839Z","shell.execute_reply":"2023-04-05T17:38:16.489945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Checking if there are any missing or null values present in the dataset.\ndata_set.isnull().any()\n\nnRow, nCol = data_set.shape\nprint(f'There are {nRow} rows and {nCol} columns')\n\ndata_set.dtypes\n\ndata_set.info()\n\ndata_set.describe(include=['object'])\n\ntypes = np.unique(np.array(data_set['type']))\ntypes\n\ntotal = data_set.groupby(['type']).count()*50\ntotal\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:16.493755Z","iopub.execute_input":"2023-04-05T17:38:16.494462Z","iopub.status.idle":"2023-04-05T17:38:16.571748Z","shell.execute_reply.started":"2023-04-05T17:38:16.494423Z","shell.execute_reply":"2023-04-05T17:38:16.570603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_text(df, remove_special=True):\n    texts = df['posts'].copy()\n    labels = df['type'].copy()\n\n    #Remove links \n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'https?:\\/\\/.*?[\\s+]', '', x.replace(\"|\",\" \") + \" \"))\n    \n    #Keep the End Of Sentence characters\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'\\.', ' EOSTokenDot ', x + \" \"))\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'\\?', ' EOSTokenQuest ', x + \" \"))\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'!', ' EOSTokenExs ', x + \" \"))\n    \n    #Strip Punctation\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'[\\.+]', \".\",x))\n\n    #Remove multiple fullstops\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'[^\\w\\s]','',x))\n\n    #Remove Non-words\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'[^a-zA-Z\\s]','',x))\n\n    #Convert posts to lowercase\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: x.lower())\n\n    #Remove multiple letter repeating words\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'([a-z])\\1{2,}[\\s|\\w]*','',x)) \n\n    #Remove very short or long words\n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'(\\b\\w{0,3})?\\b','',x)) \n    df[\"posts\"] = df[\"posts\"].apply(lambda x: re.sub(r'(\\b\\w{30,1000})?\\b','',x))\n\n    #Remove MBTI Personality Words - crutial in order to get valid model accuracy estimation for unseen data. \n    if remove_special:\n        pers_types = ['INFP' ,'INFJ', 'INTP', 'INTJ', 'ENTP', 'ENFP', 'ISTP' ,'ISFP' ,'ENTJ', 'ISTJ','ENFJ', 'ISFJ' ,'ESTP', 'ESFP' ,'ESFJ' ,'ESTJ']\n        pers_types = [p.lower() for p in pers_types]\n        p = re.compile(\"(\" + \"|\".join(pers_types) + \")\")\n    \n    return df\n\n#Preprocessing of entered Text\nnew_df = preprocess_text(data_set)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:16.573428Z","iopub.execute_input":"2023-04-05T17:38:16.573746Z","iopub.status.idle":"2023-04-05T17:38:36.956411Z","shell.execute_reply.started":"2023-04-05T17:38:16.573715Z","shell.execute_reply":"2023-04-05T17:38:36.955448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Remove posts with less than X words\nmin_words = 15\nprint(\"Before : Number of posts\", len(new_df)) \nnew_df[\"no. of. words\"] = new_df[\"posts\"].apply(lambda x: len(re.findall(r'\\w+', x)))\nnew_df = new_df[new_df[\"no. of. words\"] >= min_words]\n\nprint(\"After : Number of posts\", len(new_df))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:36.960205Z","iopub.execute_input":"2023-04-05T17:38:36.960856Z","iopub.status.idle":"2023-04-05T17:38:38.132129Z","shell.execute_reply.started":"2023-04-05T17:38:36.960802Z","shell.execute_reply":"2023-04-05T17:38:38.130869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#Feature\nnew_df.tail(15)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:38.135412Z","iopub.execute_input":"2023-04-05T17:38:38.135749Z","iopub.status.idle":"2023-04-05T17:38:38.148205Z","shell.execute_reply.started":"2023-04-05T17:38:38.135718Z","shell.execute_reply":"2023-04-05T17:38:38.147191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ignore noise warning\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# Converting MBTI personality (or target or Y feature) into numerical form using Label Encoding\n# encoding personality type\nenc = LabelEncoder()\nnew_df['type of encoding'] = enc.fit_transform(new_df['type'])\n\ntarget = new_df['type of encoding'] ","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:38.149458Z","iopub.execute_input":"2023-04-05T17:38:38.149749Z","iopub.status.idle":"2023-04-05T17:38:38.164108Z","shell.execute_reply.started":"2023-04-05T17:38:38.14972Z","shell.execute_reply":"2023-04-05T17:38:38.163148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:38.165581Z","iopub.execute_input":"2023-04-05T17:38:38.166484Z","iopub.status.idle":"2023-04-05T17:38:38.181215Z","shell.execute_reply.started":"2023-04-05T17:38:38.166449Z","shell.execute_reply":"2023-04-05T17:38:38.180125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The python natural language toolkit library provides a list of english stop words.\nprint(stopwords.words('english'))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:38.182358Z","iopub.execute_input":"2023-04-05T17:38:38.182644Z","iopub.status.idle":"2023-04-05T17:38:38.192073Z","shell.execute_reply.started":"2023-04-05T17:38:38.182615Z","shell.execute_reply":"2023-04-05T17:38:38.19084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vectorizing the posts for the model and filtering Stop-words\nvect = CountVectorizer(stop_words='english') \n\n# Converting posts (or training or X feature) into numerical form by count vectorization\ntrain =  vect.fit_transform(new_df[\"posts\"])","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:38.193293Z","iopub.execute_input":"2023-04-05T17:38:38.1941Z","iopub.status.idle":"2023-04-05T17:38:41.702069Z","shell.execute_reply.started":"2023-04-05T17:38:38.194065Z","shell.execute_reply":"2023-04-05T17:38:41.700726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:41.703533Z","iopub.execute_input":"2023-04-05T17:38:41.703981Z","iopub.status.idle":"2023-04-05T17:38:41.714731Z","shell.execute_reply.started":"2023-04-05T17:38:41.703935Z","shell.execute_reply":"2023-04-05T17:38:41.713351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.4, stratify=target, random_state=42)\nprint ((X_train.shape),(y_train.shape),(X_test.shape),(y_test.shape))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:41.7165Z","iopub.execute_input":"2023-04-05T17:38:41.716875Z","iopub.status.idle":"2023-04-05T17:38:41.736091Z","shell.execute_reply.started":"2023-04-05T17:38:41.716837Z","shell.execute_reply":"2023-04-05T17:38:41.734896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg = LogisticRegression(solver='liblinear')\nlogreg.fit(X_train, y_train)\n\nY_pred = logreg.predict(X_test)\npredictions = [round(value) for value in Y_pred]","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:38:41.737674Z","iopub.execute_input":"2023-04-05T17:38:41.738582Z","iopub.status.idle":"2023-04-05T17:39:06.968274Z","shell.execute_reply.started":"2023-04-05T17:38:41.738533Z","shell.execute_reply":"2023-04-05T17:39:06.966874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#enc.inverse_transform(new_df['type'])\npred_cat = enc.inverse_transform(new_df['type of encoding'])\n\npred = pd.DataFrame({'Category': pred_cat})\npred['Id'] = test['Id']","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:39:06.97041Z","iopub.execute_input":"2023-04-05T17:39:06.971404Z","iopub.status.idle":"2023-04-05T17:39:06.981574Z","shell.execute_reply.started":"2023-04-05T17:39:06.971343Z","shell.execute_reply":"2023-04-05T17:39:06.980347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred.to_csv(\"nguyenV7.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:39:26.721464Z","iopub.execute_input":"2023-04-05T17:39:26.7219Z","iopub.status.idle":"2023-04-05T17:39:26.737697Z","shell.execute_reply.started":"2023-04-05T17:39:26.721859Z","shell.execute_reply":"2023-04-05T17:39:26.736494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:39:07.010535Z","iopub.execute_input":"2023-04-05T17:39:07.011445Z","iopub.status.idle":"2023-04-05T17:39:07.027948Z","shell.execute_reply.started":"2023-04-05T17:39:07.011397Z","shell.execute_reply":"2023-04-05T17:39:07.026781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## form a csv file that has 2 cols: Id and Category\n### first inverse transform the prediction\n#pred_cat = le.inverse_transform(['type'])\n\n#data['Category'] = data['Category'].map({O:\"ENFJ\", 1:\"ENFP\",2:\"ENTJ\",3:\"ENTP\",4:\"ESFJ\",5:\"ESFP\",6:\"ESTJ\", 7: \"ESTP\",8:\"INFJ\", 9:\"INFP\", 10:\"INTJ\",11: \"INTP\", 12:\"ISFJ\", 13:\"ISFP\", 14: \"ISTJ\", 15:\"ISTP\"})\n#pred['Id'] = test['Id']\n\n#pred = pd.DataFrame.map({O:\"ENFJ\", 1:\"ENFP\",2:\"ENTJ\",3:\"ENTP\",4:\"ESFJ\",5:\"ESFP\",6:\"ESTJ\", 7: \"ESTP\",8:\"INFJ\", 9:\"INFP\", 10:\"INTJ\",11: \"INTP\", 12:\"ISFJ\", 13:\"ISFP\", 14: \"ISTJ\", 15:\"ISTP\"})\n#pred = pd.DataFrame({\"Category\": pred_cat})\n#pred['Id'] = test['Id']\n#y_test = data_set['type']\n#pred = pd.DataFrame({'Category', pred})\n#pred['Id'] = test['Id']\n#pred.to_csv('pred.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T17:39:07.033262Z","iopub.execute_input":"2023-04-05T17:39:07.034164Z","iopub.status.idle":"2023-04-05T17:39:07.040107Z","shell.execute_reply.started":"2023-04-05T17:39:07.034114Z","shell.execute_reply":"2023-04-05T17:39:07.038897Z"},"trusted":true},"execution_count":null,"outputs":[]}]}