{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87232,"databundleVersionId":9912598,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Library","metadata":{}},{"cell_type":"code","source":"import json\nimport pandas as pd\nimport numpy as np\nimport nltk\nimport matplotlib.pyplot as plt\nfrom collections import Counter\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report","metadata":{"_uuid":"add12f9f-a65d-45b3-9c56-de23e031498c","_cell_guid":"3a1d97ea-2d1c-4398-8ad8-2396a999f003","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:29:53.608840Z","iopub.execute_input":"2024-11-29T14:29:53.609756Z","iopub.status.idle":"2024-11-29T14:29:53.615596Z","shell.execute_reply.started":"2024-11-29T14:29:53.609709Z","shell.execute_reply":"2024-11-29T14:29:53.614604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"# Read data\ndata = []\nwith open('/kaggle/input/dm-2024-isa-5810-lab-2-homework/tweets_DM.json', 'r') as f:\n    for line in f:\n        data.append(json.loads(line))\n \nf.close()\nemotion = pd.read_csv('/kaggle/input/dm-2024-isa-5810-lab-2-homework/emotion.csv')\ndata_identification = pd.read_csv('/kaggle/input/dm-2024-isa-5810-lab-2-homework/data_identification.csv')","metadata":{"_uuid":"adb5a68b-d001-49f5-ba19-ebf2ca7d7114","_cell_guid":"24d63e07-cf6f-4ed3-8dc6-be6ed9ed867f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-30T08:28:49.338867Z","iopub.execute_input":"2024-11-30T08:28:49.339603Z","iopub.status.idle":"2024-11-30T08:28:49.369246Z","shell.execute_reply.started":"2024-11-30T08:28:49.339569Z","shell.execute_reply":"2024-11-30T08:28:49.368291Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build a new dataframe by selecting useful features.","metadata":{}},{"cell_type":"code","source":"# Filter out the 'train' data\ndf = pd.DataFrame(data)\n_source = df['_source'].apply(lambda x: x['tweet'])\ndf = pd.DataFrame({\n    'tweet_id': _source.apply(lambda x: x['tweet_id']),\n    'hashtags': _source.apply(lambda x: x['hashtags']),\n    'text': _source.apply(lambda x: x['text']),\n})\ndf = df.merge(data_identification, on='tweet_id', how='left')\n\ntrain_data = df[df['identification'] == 'train']","metadata":{"_uuid":"d89983c2-1181-4129-ae93-3080d1112dba","_cell_guid":"dd099707-578e-4510-a78d-56b969921d45","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-30T08:28:28.341219Z","iopub.execute_input":"2024-11-30T08:28:28.341596Z","iopub.status.idle":"2024-11-30T08:28:28.368397Z","shell.execute_reply.started":"2024-11-30T08:28:28.341565Z","shell.execute_reply":"2024-11-30T08:28:28.366657Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"code","source":"train_data = train_data.merge(emotion, on='tweet_id', how='left') # Merge emotion for corresponding tweet_id\ntrain_data.drop_duplicates(subset=['text'], keep=False, inplace=True) # Remove duplication","metadata":{"_uuid":"7475483a-ac22-49b0-b8f9-2c59590889be","_cell_guid":"301c2874-42b5-4cfe-8360-73e5ba4e227c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:31:10.373056Z","iopub.execute_input":"2024-11-29T14:31:10.373376Z","iopub.status.idle":"2024-11-29T14:31:13.554791Z","shell.execute_reply.started":"2024-11-29T14:31:10.373347Z","shell.execute_reply":"2024-11-29T14:31:13.554090Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_sample = train_data.sample(frac=0.1) # Get sample","metadata":{"_uuid":"d14f42ef-dd9b-4f0e-9df6-4d55c8a9cb6e","_cell_guid":"da133c00-1c63-4df9-8b19-63f20214ed73","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:31:26.940623Z","iopub.execute_input":"2024-11-29T14:31:26.941260Z","iopub.status.idle":"2024-11-29T14:31:27.106654Z","shell.execute_reply.started":"2024-11-29T14:31:26.941225Z","shell.execute_reply":"2024-11-29T14:31:27.105932Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:31:34.114217Z","iopub.execute_input":"2024-11-29T14:31:34.114945Z","iopub.status.idle":"2024-11-29T14:31:34.127519Z","shell.execute_reply.started":"2024-11-29T14:31:34.114909Z","shell.execute_reply":"2024-11-29T14:31:34.126678Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Droping tweet_id and identification, because it isn't going to be features used on training","metadata":{}},{"cell_type":"code","source":"y_train_data = train_data_sample['emotion']\nX_train_data = train_data_sample.drop(['tweet_id', 'emotion', 'identification'], axis=1)\nX_train_data = X_train_data['text'] + ' ' + X_train_data['hashtags'].apply(lambda x: ' '.join(x)) # Combine text and hashtags ","metadata":{"_uuid":"9a987df6-b0d2-427c-80ff-a8cd876c6e96","_cell_guid":"39937cf7-dddd-41d9-aaa8-09274dd3ef19","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:31:38.687791Z","iopub.execute_input":"2024-11-29T14:31:38.688169Z","iopub.status.idle":"2024-11-29T14:31:38.876522Z","shell.execute_reply.started":"2024-11-29T14:31:38.688139Z","shell.execute_reply":"2024-11-29T14:31:38.875853Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:31:40.463779Z","iopub.execute_input":"2024-11-29T14:31:40.464152Z","iopub.status.idle":"2024-11-29T14:31:40.471388Z","shell.execute_reply.started":"2024-11-29T14:31:40.464123Z","shell.execute_reply":"2024-11-29T14:31:40.470462Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### For the current stage, we now have combination of text and hashtags.","metadata":{}},{"cell_type":"markdown","source":"### ","metadata":{}},{"cell_type":"code","source":"# This part is optional\n\n# Instead of using whole word, I want to use only adjective word for training, so I use spacy for filtering.\n\n# Use spacy for filtering adjective\nimport spacy\nnlp = spacy.load('en_core_web_sm')\nnlp\n\nimport pandas as pd\n\ndef keep_only_adjectives_spacy(texts):\n    filtered_texts = []\n    for text in texts:\n        doc = nlp(text)\n        # Filter\n        adjectives = [token.text for token in doc if token.pos_ == 'ADJ']\n\n        filtered_texts.append(' '.join(adjectives))\n    return filtered_texts\n\n# Get the adjective words\nX_train_data = keep_only_adjectives_spacy(X_train_data)\n\nprint(X_train_data)\n\n# Some of rows do not have adjective, so I just remove it.\nfiltered_X_train_data = []\nfiltered_y_train_data = []\n\nfor text, emotion in zip(X_train_data, y_train_data):\n    if text:  # only keep non-empty text\n        filtered_X_train_data.append(text)\n        filtered_y_train_data.append(emotion)\n\nprint(len(filtered_X_train_data))\nprint(len(filtered_y_train_data))\n\nX_train_data = filtered_X_train_data\ny_train_data = filtered_y_train_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T08:13:13.698570Z","iopub.execute_input":"2024-11-30T08:13:13.698970Z","iopub.status.idle":"2024-11-30T08:13:14.359969Z","shell.execute_reply.started":"2024-11-30T08:13:13.698942Z","shell.execute_reply":"2024-11-30T08:13:14.358843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n# Split training and testing data for evaluation.\nX_train, X_test, y_train, y_test = train_test_split(X_train_data, y_train_data, test_size=0.2, random_state=42, stratify=y_train_data) \n","metadata":{"_uuid":"783a7801-2b7b-48af-bc79-daa7683d8e9d","_cell_guid":"27d6f1a6-e2c1-4f02-be9c-b0a941f7d4d3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:21:42.414707Z","iopub.execute_input":"2024-11-29T14:21:42.415095Z","iopub.status.idle":"2024-11-29T14:21:42.468495Z","shell.execute_reply.started":"2024-11-29T14:21:42.415062Z","shell.execute_reply":"2024-11-29T14:21:42.467514Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tfidf = TfidfVectorizer(max_features=1000) # Use tfidfVectorizer and remove stop_words.\nX = tfidf.fit_transform(X_train).toarray()\nX_test = tfidf.transform(X_test)","metadata":{"_uuid":"8c24dec5-49fb-466a-9261-c8c32aa4a651","_cell_guid":"69a49263-23bf-4c77-80b4-599109e07eb4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:55:14.253150Z","iopub.execute_input":"2024-11-29T14:55:14.253980Z","iopub.status.idle":"2024-11-29T14:55:15.259102Z","shell.execute_reply.started":"2024-11-29T14:55:14.253945Z","shell.execute_reply":"2024-11-29T14:55:15.258334Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"le = LabelEncoder() # Label target\ny = le.fit_transform(y_train)\ny_test = le.transform(y_test)","metadata":{"_uuid":"9f0721ae-12d7-4664-8abb-c2bb624517c4","_cell_guid":"4cd63df9-a7b2-4855-ac8e-d6b76081018c","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:55:20.117276Z","iopub.execute_input":"2024-11-29T14:55:20.118120Z","iopub.status.idle":"2024-11-29T14:55:20.150359Z","shell.execute_reply.started":"2024-11-29T14:55:20.118084Z","shell.execute_reply":"2024-11-29T14:55:20.149464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clf = RandomForestClassifier() # Use RandomForest model\nclf.fit(X, y)\nmodel = clf","metadata":{"_uuid":"f736b5f0-b152-4af2-afd3-1000ddc9ae8b","_cell_guid":"743b776c-f14b-4bda-910e-67a23b359bd5","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:55:23.481659Z","iopub.execute_input":"2024-11-29T14:55:23.482356Z","iopub.status.idle":"2024-11-29T15:53:43.250556Z","shell.execute_reply.started":"2024-11-29T14:55:23.482319Z","shell.execute_reply":"2024-11-29T15:53:43.249520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test) # Predict","metadata":{"_uuid":"55b7553c-bd26-430a-9b84-99441bd41e94","_cell_guid":"88f87726-0018-49e1-8bf3-d459da0636e4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T16:05:25.473044Z","iopub.execute_input":"2024-11-29T16:05:25.473384Z","iopub.status.idle":"2024-11-29T16:05:25.636677Z","shell.execute_reply.started":"2024-11-29T16:05:25.473354Z","shell.execute_reply":"2024-11-29T16:05:25.635409Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test, y_pred) # Evaluation","metadata":{"_uuid":"1d21fd9c-03b7-4e10-953a-f437738adf48","_cell_guid":"c7146926-5ba4-4bf7-8535-eb464dab232d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:24:59.639057Z","iopub.execute_input":"2024-11-29T14:24:59.639619Z","iopub.status.idle":"2024-11-29T14:24:59.646648Z","shell.execute_reply.started":"2024-11-29T14:24:59.639584Z","shell.execute_reply":"2024-11-29T14:24:59.645878Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Deal with test data","metadata":{"_uuid":"c459c037-8030-4474-bde1-be24e57d296b","_cell_guid":"18388020-30e4-4a7a-8ff2-70aa0b08b93d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"### Convert test data into the form that same as train data.","metadata":{}},{"cell_type":"code","source":"test_data = df[df['identification'] == 'test'] # Get the test data from df","metadata":{"_uuid":"f0f83c94-2b60-4fea-9458-7b4122200281","_cell_guid":"1d567e2c-dc17-4229-9b6b-01fbe4e2189c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T16:06:02.964275Z","iopub.execute_input":"2024-11-29T16:06:02.964907Z","iopub.status.idle":"2024-11-29T16:06:03.276958Z","shell.execute_reply.started":"2024-11-29T16:06:02.964869Z","shell.execute_reply":"2024-11-29T16:06:03.275986Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Do the same thing as training stage, but here we don't have emotions feature.\nX_test_data = test_data.drop(['tweet_id', 'identification'], axis=1)\nX_test_data = X_test_data['text'] + ' ' + X_test_data['hashtags'].apply(lambda x: ' '.join(x))","metadata":{"_uuid":"a48cff73-74c7-456b-9f91-517bab107e8e","_cell_guid":"73ed8768-8b5c-4bb9-9c86-4f374bb1e530","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T16:06:04.848605Z","iopub.execute_input":"2024-11-29T16:06:04.848962Z","iopub.status.idle":"2024-11-29T16:06:05.374727Z","shell.execute_reply.started":"2024-11-29T16:06:04.848930Z","shell.execute_reply":"2024-11-29T16:06:05.374024Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test_data = tfidf.transform(X_test_data).toarray() # Convert test data by using same tfidfVectorizer","metadata":{"_uuid":"f37af5cb-47db-443b-b6af-c235b2bc9f6d","_cell_guid":"17f85863-fcc4-4564-ab61-efd8f336ac30","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-11-29T16:06:07.503856Z","iopub.execute_input":"2024-11-29T16:06:07.504164Z","iopub.status.idle":"2024-11-29T16:06:25.848443Z","shell.execute_reply.started":"2024-11-29T16:06:07.504136Z","shell.execute_reply":"2024-11-29T16:06:25.847704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_pred = model.predict(X_test_data)","metadata":{"_uuid":"3309b545-1af4-4fc6-82d6-386f148e9aa6","_cell_guid":"fcd028a4-172e-4542-b934-287bddadc62d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T16:06:58.910871Z","iopub.execute_input":"2024-11-29T16:06:58.911671Z","iopub.status.idle":"2024-11-29T16:06:59.136676Z","shell.execute_reply.started":"2024-11-29T16:06:58.911632Z","shell.execute_reply":"2024-11-29T16:06:59.135567Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_labels = le.inverse_transform(y_test_pred) # Inverse predict labels back to adjective words","metadata":{"_uuid":"e43ae83f-628e-4aee-bfd9-9ab48890d4b5","_cell_guid":"2eba1bff-883d-4fe0-867d-684c6f66fbc2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:27:36.022499Z","iopub.execute_input":"2024-11-29T14:27:36.023198Z","iopub.status.idle":"2024-11-29T14:27:36.036314Z","shell.execute_reply.started":"2024-11-29T14:27:36.023166Z","shell.execute_reply":"2024-11-29T14:27:36.035563Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Make up submit format","metadata":{"_uuid":"556454d0-f4b9-4565-b422-3538fbede0c6","_cell_guid":"515e5b52-c6aa-4e11-95d1-0a617e8969f1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'id': test_data['tweet_id'],\n    'emotion': y_pred_labels\n})","metadata":{"_uuid":"13a14210-bddb-40cf-b0d6-107d1f0af0a8","_cell_guid":"c831f309-182a-4377-b100-d7e67ce3d6ba","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:27:38.113417Z","iopub.execute_input":"2024-11-29T14:27:38.114273Z","iopub.status.idle":"2024-11-29T14:27:38.183953Z","shell.execute_reply.started":"2024-11-29T14:27:38.114237Z","shell.execute_reply":"2024-11-29T14:27:38.183209Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"_uuid":"c31c854c-fab5-46f4-82ca-eb9e0e641faa","_cell_guid":"5631bd37-f72e-4190-9214-8829f83cad35","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T14:27:39.894646Z","iopub.execute_input":"2024-11-29T14:27:39.895014Z","iopub.status.idle":"2024-11-29T14:27:40.275404Z","shell.execute_reply.started":"2024-11-29T14:27:39.894982Z","shell.execute_reply":"2024-11-29T14:27:40.274718Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}