{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf=pd.read_csv('/kaggle/input/emotions-in-text/Emotion_final.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:55:53.829205Z","iopub.execute_input":"2023-06-15T09:55:53.830241Z","iopub.status.idle":"2023-06-15T09:55:53.952427Z","shell.execute_reply.started":"2023-06-15T09:55:53.830186Z","shell.execute_reply":"2023-06-15T09:55:53.950834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:55:59.171295Z","iopub.execute_input":"2023-06-15T09:55:59.172270Z","iopub.status.idle":"2023-06-15T09:55:59.247159Z","shell.execute_reply.started":"2023-06-15T09:55:59.172215Z","shell.execute_reply":"2023-06-15T09:55:59.245701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Extract and separate the data based on their labels\nfrom sklearn import preprocessing\n\nlabel_encoder=preprocessing.LabelEncoder()\ndf['Emotion']=label_encoder.fit_transform(df['Emotion'])\ntrain=df","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:56:31.066758Z","iopub.execute_input":"2023-06-15T09:56:31.067268Z","iopub.status.idle":"2023-06-15T09:56:31.272666Z","shell.execute_reply.started":"2023-06-15T09:56:31.067226Z","shell.execute_reply":"2023-06-15T09:56:31.271192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data cleaning**","metadata":{}},{"cell_type":"code","source":"#Remove Hashtags\ntrain['Text'].replace({ r\"#(\\w+)\" : '' },inplace=True,regex=True)\n\n#Remove Mention\ntrain['Text'].replace({ r\"@(\\w+)\" : '' },inplace=True,regex=True)\n\n#Remove URL\ntrain['Text'].astype(str).replace({ r\"http\\S+\" : '' },inplace=True,regex=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:56:39.627385Z","iopub.execute_input":"2023-06-15T09:56:39.627855Z","iopub.status.idle":"2023-06-15T09:56:39.790753Z","shell.execute_reply.started":"2023-06-15T09:56:39.627812Z","shell.execute_reply":"2023-06-15T09:56:39.789231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**converting text to small case**","metadata":{}},{"cell_type":"code","source":"train['Text']=train['Text'].str.lower()\ndf_train=train","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:14:58.293614Z","iopub.execute_input":"2023-06-15T10:14:58.294081Z","iopub.status.idle":"2023-06-15T10:14:58.317237Z","shell.execute_reply.started":"2023-06-15T10:14:58.294035Z","shell.execute_reply":"2023-06-15T10:14:58.315730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Stopword Removal**","metadata":{}},{"cell_type":"code","source":"# Import stopwords with nltk.\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\n\ntrain['Text'] = train['Text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:56:54.108104Z","iopub.execute_input":"2023-06-15T09:56:54.108524Z","iopub.status.idle":"2023-06-15T09:56:55.846863Z","shell.execute_reply.started":"2023-06-15T09:56:54.108488Z","shell.execute_reply":"2023-06-15T09:56:55.845301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TEXT VECTORIZATION**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ndf_t=train['Text'].values\ny_t=train['Emotion'].values\n\n# splitting the training and testing part from the data\nX_tr, X_va, Y_tr, Y_va = train_test_split(df_t,y_t, test_size=0.2, random_state=0)\n\n\n\nprint(\"Training data:\", X_tr.shape, Y_tr.shape)\nprint(\"Validation data:\", X_va.shape, Y_va.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:57:06.632268Z","iopub.execute_input":"2023-06-15T09:57:06.632711Z","iopub.status.idle":"2023-06-15T09:57:06.647598Z","shell.execute_reply.started":"2023-06-15T09:57:06.632672Z","shell.execute_reply":"2023-06-15T09:57:06.645983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\n# Tokenize and pad the sequences\ntokenizer = Tokenizer(num_words=20000)\ntokenizer.fit_on_texts(X_tr)\n\nmax_seq_length = 250\n\nX_tr_seq = tokenizer.texts_to_sequences(X_tr)\nX_tr_seq = pad_sequences(X_tr_seq, maxlen=max_seq_length)\n\nX_va_seq = tokenizer.texts_to_sequences(X_va)\nX_va_seq = pad_sequences(X_va_seq, maxlen=max_seq_length)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:57:22.528896Z","iopub.execute_input":"2023-06-15T09:57:22.529406Z","iopub.status.idle":"2023-06-15T09:57:34.303938Z","shell.execute_reply.started":"2023-06-15T09:57:22.529359Z","shell.execute_reply":"2023-06-15T09:57:34.302326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Applying the models**","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn import tree\n\n# For creating a table of the accuracies in the end\naccuracy = {'TF-IDF':[]}\n\n# Logistic Regression\nregressor_LR_tf = LogisticRegression(C=1.0,penalty='l2',solver='newton-cg')\nregressor_LR_tf.fit(X_tr_seq, Y_tr)\ny_predict_LR_tf = regressor_LR_tf.predict(X_va_seq)\na=(regressor_LR_tf.score(X_va_seq, Y_va))\naccuracy['TF-IDF'].append(a)\n\n# Decision Tree\nmodel_DT_tf = tree.DecisionTreeClassifier(criterion = 'gini', max_depth=2) \nmodel_DT_tf.fit(X_tr_seq, Y_tr)\ny_predict_DT_tf = model_DT_tf.predict(X_va_seq)\na=(model_DT_tf.score(X_va_seq,Y_va))\naccuracy['TF-IDF'].append(a)\n\n# K-Neighbors Classifier\nmodel_KN_tf = KNeighborsClassifier(metric= 'manhattan', n_neighbors= 5, weights= 'distance') \nmodel_KN_tf.fit(X_tr_seq, Y_tr)\ny_predict_KN_tf = model_KN_tf.predict(X_va_seq)\na=(model_KN_tf.score(X_va_seq,Y_va))\naccuracy['TF-IDF'].append(a)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:09:59.920575Z","iopub.execute_input":"2023-06-15T10:09:59.922163Z","iopub.status.idle":"2023-06-15T10:14:10.830643Z","shell.execute_reply.started":"2023-06-15T10:09:59.922111Z","shell.execute_reply":"2023-06-15T10:14:10.829168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\nmodel = ['LR','DT','KN']\ndata = {'model':model,'accuracy':accuracy['TF-IDF']}\ncompare_models = pd.DataFrame(data)\ncompare_models","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:14:31.966443Z","iopub.execute_input":"2023-06-15T10:14:31.966910Z","iopub.status.idle":"2023-06-15T10:14:31.981898Z","shell.execute_reply.started":"2023-06-15T10:14:31.966869Z","shell.execute_reply":"2023-06-15T10:14:31.980344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get 50 most frequent words in the training set\nimport matplotlib.pyplot as plt\n\nfrequent_words = pd.Series(' '.join(df_train.Text).split()).value_counts()[:50]\nfrequent_words.plot.bar(figsize=(15,4))\nplt.title('50 most frequent terms in the training set', fontweight=\"bold\")\nplt.xlabel('Tokens')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:15:04.458284Z","iopub.execute_input":"2023-06-15T10:15:04.458724Z","iopub.status.idle":"2023-06-15T10:15:05.500245Z","shell.execute_reply.started":"2023-06-15T10:15:04.458677Z","shell.execute_reply":"2023-06-15T10:15:05.498744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"the Keras module from TensorFlow to preprocess text data. Specifically, it uses the 'Tokenizer' function to convert the text into a sequence of integers, and the 'pad_sequences' function to ensure that each sequence has the same length.","metadata":{}},{"cell_type":"markdown","source":"**Using Neural LSTM**","metadata":{}},{"cell_type":"markdown","source":"There are various approaches available, but I decided to experiment with a combination of several different layers, including a convolutional layer and a bidirectional LTSM layers. Probably these ones may be responsible for slowing down the training time more than the other layers","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Embedding, Conv1D, MaxPooling1D, LSTM, Bidirectional, Dense, BatchNormalization, Dropout\nfrom tensorflow.keras.optimizers import Adam\n\n# with strategy.scope():\nmodel = Sequential()\nmodel.add(Embedding(input_dim=20000, output_dim=128, input_length=250))\n\nmodel.add(Conv1D(filters=64, kernel_size=3, activation='relu'))\nmodel.add(MaxPooling1D(pool_size=2))\n\nmodel.add(Bidirectional(LSTM(units=64, return_sequences=True)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Bidirectional(LSTM(units=128)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(units=32, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(units=6, activation='softmax'))\n\nmodel.compile(optimizer=Adam(learning_rate = 1e-4),\n                  loss = 'sparse_categorical_crossentropy',\n                  metrics=['acc']\n                 )\n\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:15:44.614384Z","iopub.execute_input":"2023-06-15T10:15:44.614834Z","iopub.status.idle":"2023-06-15T10:15:46.356395Z","shell.execute_reply.started":"2023-06-15T10:15:44.614796Z","shell.execute_reply":"2023-06-15T10:15:46.355012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\n\n# Creates 'EarlyStopping' callback\nearlystopping_cb = EarlyStopping(patience=4, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:15:57.611529Z","iopub.execute_input":"2023-06-15T10:15:57.612044Z","iopub.status.idle":"2023-06-15T10:15:57.621134Z","shell.execute_reply.started":"2023-06-15T10:15:57.611945Z","shell.execute_reply":"2023-06-15T10:15:57.619214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_tr_seq,\n                    Y_tr,\n                    validation_data=(X_va_seq, Y_va),\n                    callbacks=[earlystopping_cb],\n                    batch_size=64,\n                    epochs=7,\n                    verbose=1,\n                   )","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:40:58.866988Z","iopub.execute_input":"2023-06-03T05:40:58.867517Z","iopub.status.idle":"2023-06-03T05:57:30.819504Z","shell.execute_reply.started":"2023-06-03T05:40:58.867471Z","shell.execute_reply":"2023-06-03T05:57:30.818235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l, test_acc = model.evaluate(X_va_seq,Y_va)\n\nprint('Validation loss:', l)\nprint('Validation accuracy:', test_acc)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:57:52.657045Z","iopub.execute_input":"2023-06-03T05:57:52.657499Z","iopub.status.idle":"2023-06-03T05:58:06.806989Z","shell.execute_reply.started":"2023-06-03T05:57:52.657457Z","shell.execute_reply":"2023-06-03T05:58:06.805971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make predictions**","metadata":{}},{"cell_type":"code","source":"import numpy as np\npredictions = [np.argmax(i) for i in model.predict(X_va_seq)]\n# Create a new DataFrame to merge review ids and the model predictions\nsubmission = pd.DataFrame({'Emotion': predictions})","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:58:11.959995Z","iopub.execute_input":"2023-06-03T05:58:11.960461Z","iopub.status.idle":"2023-06-03T05:58:28.012707Z","shell.execute_reply.started":"2023-06-03T05:58:11.960418Z","shell.execute_reply":"2023-06-03T05:58:28.011644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nsubmission['Emotion'].value_counts().sort_index().plot.bar(figsize=(10,5))\nplt.title('Emotion type distribution of the predictions',fontweight=\"bold\")\nplt.xlabel('Emotion')\nplt.ylabel('Number of Text')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:58:55.061461Z","iopub.execute_input":"2023-06-03T05:58:55.061881Z","iopub.status.idle":"2023-06-03T05:58:55.326019Z","shell.execute_reply.started":"2023-06-03T05:58:55.061845Z","shell.execute_reply":"2023-06-03T05:58:55.324621Z"},"trusted":true},"execution_count":null,"outputs":[]}]}