{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":87232,"databundleVersionId":9912598,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Set up\nimport pandas as pd\nimport numpy as np\nimport json\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, f1_score","metadata":{"_uuid":"242160c4-2a74-45e1-9258-9f3a36c480d8","_cell_guid":"0f9bfb18-b8eb-4d0c-9f90-0685e653aeaa","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:32.238842Z","iopub.execute_input":"2024-11-29T07:16:32.239546Z","iopub.status.idle":"2024-11-29T07:16:33.404225Z","shell.execute_reply.started":"2024-11-29T07:16:32.239497Z","shell.execute_reply":"2024-11-29T07:16:33.403534Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Load the data\n#Understand the data_identification first\ndata_identification = pd.read_csv('/kaggle/input/dm-2024-isa-5810-lab-2-homework/data_identification.csv')","metadata":{"_uuid":"71385a0f-fdb0-42a3-ba08-7809ffa8c24d","_cell_guid":"15f87024-3f7c-4d6e-9820-6b0fe9e59370","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:33.405740Z","iopub.execute_input":"2024-11-29T07:16:33.406125Z","iopub.status.idle":"2024-11-29T07:16:34.660929Z","shell.execute_reply.started":"2024-11-29T07:16:33.406081Z","shell.execute_reply":"2024-11-29T07:16:34.660251Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_identification.head())","metadata":{"_uuid":"af055ce1-8c80-41da-9fab-c3d27d979df2","_cell_guid":"9d07334b-05f0-4554-8e77-9d33234604d6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:34.661823Z","iopub.execute_input":"2024-11-29T07:16:34.662047Z","iopub.status.idle":"2024-11-29T07:16:34.672763Z","shell.execute_reply.started":"2024-11-29T07:16:34.662023Z","shell.execute_reply":"2024-11-29T07:16:34.671978Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_identification.isnull().sum())","metadata":{"_uuid":"a7d72fca-3a5a-4d00-90b6-38b46a0d18cc","_cell_guid":"2b80e6f5-7336-4fda-812e-06d4be82f14c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:34.674316Z","iopub.execute_input":"2024-11-29T07:16:34.674576Z","iopub.status.idle":"2024-11-29T07:16:34.831826Z","shell.execute_reply.started":"2024-11-29T07:16:34.674544Z","shell.execute_reply":"2024-11-29T07:16:34.830870Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_identification.info())","metadata":{"_uuid":"5afbd43c-868f-4897-9079-0f1158ff5450","_cell_guid":"cc9daefb-729b-436b-8a49-e9bc45a2f2e8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:34.832935Z","iopub.execute_input":"2024-11-29T07:16:34.833233Z","iopub.status.idle":"2024-11-29T07:16:34.853935Z","shell.execute_reply.started":"2024-11-29T07:16:34.833206Z","shell.execute_reply":"2024-11-29T07:16:34.853150Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Check the distribution\nsns.countplot(data=data_identification, x='identification')\nplt.title(\"Training vs Testing Data Distribution\")\nplt.show()","metadata":{"_uuid":"46278820-e923-4cd2-9b02-61e4bd0c476a","_cell_guid":"60d255a5-96e1-464d-93e3-68b7a28cd8b7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:34.855188Z","iopub.execute_input":"2024-11-29T07:16:34.855513Z","iopub.status.idle":"2024-11-29T07:16:35.980179Z","shell.execute_reply.started":"2024-11-29T07:16:34.855476Z","shell.execute_reply":"2024-11-29T07:16:35.979337Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Number of duplicate tweet_id: {data_identification['tweet_id'].duplicated().sum()}\")","metadata":{"_uuid":"bcb41c36-e01c-4fb6-9cdf-1b611371e539","_cell_guid":"83dcfe96-36c6-42c2-a424-6b5f9abf5aa2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:35.981350Z","iopub.execute_input":"2024-11-29T07:16:35.981694Z","iopub.status.idle":"2024-11-29T07:16:36.833026Z","shell.execute_reply.started":"2024-11-29T07:16:35.981655Z","shell.execute_reply":"2024-11-29T07:16:36.832126Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Next is tweets_DM\nwith open('/kaggle/input/dm-2024-isa-5810-lab-2-homework/tweets_DM.json', 'r') as f:\n    for i, line in enumerate(f):\n        print(line.strip())\n        if i == 4:  # Print first 5 lines\n            break","metadata":{"_uuid":"163ea3c1-8a76-4094-9617-5b698e43aa46","_cell_guid":"43bb8e45-acd2-4637-83a2-78dc66cef1c9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:36.834144Z","iopub.execute_input":"2024-11-29T07:16:36.834484Z","iopub.status.idle":"2024-11-29T07:16:36.846920Z","shell.execute_reply.started":"2024-11-29T07:16:36.834457Z","shell.execute_reply":"2024-11-29T07:16:36.846030Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Convert to dataframe\nimport json\nimport pandas as pd\n\ntweets = []\n\nwith open('/kaggle/input/dm-2024-isa-5810-lab-2-homework/tweets_DM.json', 'r') as f:\n    for line in f:\n        data = json.loads(line.strip())  \n        tweet_id = data[\"_source\"][\"tweet\"][\"tweet_id\"]\n        text = data[\"_source\"][\"tweet\"][\"text\"]\n        tweets.append({\"tweet_id\": tweet_id, \"text\": text})\n\n\ntweets_df = pd.DataFrame(tweets)\n\nprint(tweets_df.head())","metadata":{"_uuid":"61cf81d1-4c6b-4100-8e0a-a66a9e3d3da9","_cell_guid":"059d3586-bc29-49aa-b7c2-01ed103a84c3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:36.847861Z","iopub.execute_input":"2024-11-29T07:16:36.848072Z","iopub.status.idle":"2024-11-29T07:16:49.868094Z","shell.execute_reply.started":"2024-11-29T07:16:36.848050Z","shell.execute_reply":"2024-11-29T07:16:49.867176Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#emotion.csv\nemotion = pd.read_csv('/kaggle/input/dm-2024-isa-5810-lab-2-homework/emotion.csv')","metadata":{"_uuid":"c02127e7-5103-4128-957b-3dc55e2f330f","_cell_guid":"3252f9d1-a09a-4874-8eff-219a3d779f56","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:49.871741Z","iopub.execute_input":"2024-11-29T07:16:49.871989Z","iopub.status.idle":"2024-11-29T07:16:50.740093Z","shell.execute_reply.started":"2024-11-29T07:16:49.871964Z","shell.execute_reply":"2024-11-29T07:16:50.739127Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(emotion.head())\nprint(emotion.info())","metadata":{"_uuid":"42674dc0-a7cd-4d01-9b8e-91b65586067b","_cell_guid":"d8448f7b-3299-4e10-bcaf-5fd15aff062b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:50.741520Z","iopub.execute_input":"2024-11-29T07:16:50.741822Z","iopub.status.idle":"2024-11-29T07:16:50.866873Z","shell.execute_reply.started":"2024-11-29T07:16:50.741793Z","shell.execute_reply":"2024-11-29T07:16:50.866014Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Check the distribution\nsns.countplot(data=emotion, x='emotion', order=emotion['emotion'].value_counts().index)\nplt.title(\"Emotion Label Distribution\")\nplt.xticks(rotation=45)\nplt.show()","metadata":{"_uuid":"e49d2fe1-476f-46d2-b15b-e69c0ca2bb84","_cell_guid":"6711f3ee-7da5-493b-868c-4a5bd4f039e3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:50.867891Z","iopub.execute_input":"2024-11-29T07:16:50.868210Z","iopub.status.idle":"2024-11-29T07:16:51.614276Z","shell.execute_reply.started":"2024-11-29T07:16:50.868180Z","shell.execute_reply":"2024-11-29T07:16:51.613414Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Number of missing tweet_id: {emotion['tweet_id'].isnull().sum()}\")\nprint(f\"Number of duplicate tweet_id: {emotion['tweet_id'].duplicated().sum()}\")","metadata":{"_uuid":"f8bc7404-7f2a-42b9-956c-afc97c0e3c96","_cell_guid":"4f3bfd24-cd93-4d44-8b2d-88ebba20c671","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:51.615424Z","iopub.execute_input":"2024-11-29T07:16:51.615689Z","iopub.status.idle":"2024-11-29T07:16:51.965979Z","shell.execute_reply.started":"2024-11-29T07:16:51.615661Z","shell.execute_reply":"2024-11-29T07:16:51.965189Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine data_identification.csv and tweets_DM.json\ntweets_df = pd.DataFrame(tweets)\nmerged_data = pd.merge(data_identification, tweets_df, on='tweet_id', how='left')  # Merge using tweet_id\n\nprint(merged_data.head())","metadata":{"_uuid":"01593a40-3f23-4ceb-b623-1b2f9af22ffa","_cell_guid":"78bc99ba-93a7-4879-8222-93703fc4007e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:51.967062Z","iopub.execute_input":"2024-11-29T07:16:51.967352Z","iopub.status.idle":"2024-11-29T07:16:55.728259Z","shell.execute_reply.started":"2024-11-29T07:16:51.967324Z","shell.execute_reply":"2024-11-29T07:16:55.727293Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Add emotion labels to training data\ntrain_data = pd.merge(merged_data[merged_data['identification'] == 'train'], emotion, on='tweet_id', how='left')\ntest_data = merged_data[merged_data['identification'] == 'test']","metadata":{"_uuid":"29bfcde1-c73a-4d69-a33f-a0befe3eeea8","_cell_guid":"94e56fa5-a4ef-4dde-a277-3433333ff4ec","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:55.729476Z","iopub.execute_input":"2024-11-29T07:16:55.729845Z","iopub.status.idle":"2024-11-29T07:16:57.989385Z","shell.execute_reply.started":"2024-11-29T07:16:55.729803Z","shell.execute_reply":"2024-11-29T07:16:57.988412Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_data.head())\nprint(test_data.head())","metadata":{"_uuid":"2da1f489-2206-4a7e-8b17-50165afd87bd","_cell_guid":"40b3f8ef-0b8d-48b2-ab5a-816287c56fba","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-11-29T07:16:57.990513Z","iopub.execute_input":"2024-11-29T07:16:57.990766Z","iopub.status.idle":"2024-11-29T07:16:57.998589Z","shell.execute_reply.started":"2024-11-29T07:16:57.990742Z","shell.execute_reply":"2024-11-29T07:16:57.997727Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Text Preprocessing\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstop_words = set(stopwords.words('english'))\n\ndef clean_text(text):\n    text = text.lower()\n    text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text)  # Remove URLs\n    text = re.sub(r'@\\w+|#\\w+', '', text)  # Remove mentions and hashtags\n    text = re.sub(r'<lh>', '', text)  # Remove <LH> placeholders\n    text = re.sub(r'[^a-z\\s]', '', text)  # Remove special characters\n    words = text.split()\n    words = [word for word in words if word not in stop_words]  # Remove stopwords\n    return ' '.join(words)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:16:57.999569Z","iopub.execute_input":"2024-11-29T07:16:57.999824Z","iopub.status.idle":"2024-11-29T07:16:58.528244Z","shell.execute_reply.started":"2024-11-29T07:16:57.999795Z","shell.execute_reply":"2024-11-29T07:16:58.527557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = train_data.copy()  \ntest_data = test_data.copy() \n\ntrain_data.loc[:, 'cleaned_text'] = train_data['text'].apply(clean_text)\ntest_data.loc[:, 'cleaned_text'] = test_data['text'].apply(clean_text)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:16:58.529215Z","iopub.execute_input":"2024-11-29T07:16:58.529455Z","iopub.status.idle":"2024-11-29T07:17:17.308076Z","shell.execute_reply.started":"2024-11-29T07:16:58.529431Z","shell.execute_reply":"2024-11-29T07:17:17.307126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_data[['text', 'cleaned_text']].head())\nprint(test_data[['text', 'cleaned_text']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:17.309266Z","iopub.execute_input":"2024-11-29T07:17:17.309570Z","iopub.status.idle":"2024-11-29T07:17:17.509301Z","shell.execute_reply.started":"2024-11-29T07:17:17.309542Z","shell.execute_reply":"2024-11-29T07:17:17.508415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Most frequent label is \"joy,\" while \"anger\" is the least frequent.\n#This imbalance could affect model training and evaluation.\n\nfrom sklearn.utils import resample\n\n# Get the majority class\nmajority = train_data[train_data['emotion'] == 'joy']\n\n# Set a target size for oversampling\ntarget_size = int(len(majority) * 0.75)  # 75% of the majority class size\n\nbalanced_data = [majority]\n\nfor emotion in train_data['emotion'].unique():\n    if emotion != 'joy':\n        minority = train_data[train_data['emotion'] == emotion]\n        oversampled_minority = resample(minority, \n                                        replace=True, \n                                        n_samples=min(target_size, len(majority)),  # Cap size\n                                        random_state=42)\n        balanced_data.append(oversampled_minority)\n\ntrain_data_balanced = pd.concat(balanced_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:17.510546Z","iopub.execute_input":"2024-11-29T07:17:17.510920Z","iopub.status.idle":"2024-11-29T07:17:19.548705Z","shell.execute_reply.started":"2024-11-29T07:17:17.510880Z","shell.execute_reply":"2024-11-29T07:17:19.547942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check new label distribution\nsns.countplot(data=train_data_balanced, x='emotion', order=train_data_balanced['emotion'].value_counts().index)\nplt.title(\"Balanced Emotion Label Distribution\")\nplt.xticks(rotation=45)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:19.549798Z","iopub.execute_input":"2024-11-29T07:17:19.550164Z","iopub.status.idle":"2024-11-29T07:17:21.003921Z","shell.execute_reply.started":"2024-11-29T07:17:19.550103Z","shell.execute_reply":"2024-11-29T07:17:21.003178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#try use SMOTE for synthetic data\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom imblearn.over_sampling import SMOTE\nfrom collections import Counter\n\n# Vectorize the text data\nvectorizer = TfidfVectorizer(max_features=500, ngram_range=(1, 1), max_df=0.8, min_df=5)\nX_train = vectorizer.fit_transform(train_data['cleaned_text'])\ny_train = train_data['emotion']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:21.005149Z","iopub.execute_input":"2024-11-29T07:17:21.005777Z","iopub.status.idle":"2024-11-29T07:17:30.997570Z","shell.execute_reply.started":"2024-11-29T07:17:21.005735Z","shell.execute_reply":"2024-11-29T07:17:30.996873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the shape of the feature matrix\nprint(\"TF-IDF matrix shape:\", X_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:30.998552Z","iopub.execute_input":"2024-11-29T07:17:30.998977Z","iopub.status.idle":"2024-11-29T07:17:31.003841Z","shell.execute_reply.started":"2024-11-29T07:17:30.998943Z","shell.execute_reply":"2024-11-29T07:17:31.002917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Subset a smaller portion of the dataset for testing becasue of computional limitation\ntrain_sample = train_data_balanced.sample(frac=0.2, random_state=42)\nX_sample = vectorizer.transform(train_sample['cleaned_text'])\ny_sample = train_sample['emotion']\n\n# Apply SMOTE on the subset\nsmote = SMOTE(random_state=42)\nX_sample_smote, y_sample_smote = smote.fit_resample(X_sample, y_sample)\n\nprint(\"Subset SMOTE label distribution:\", Counter(y_sample_smote))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:17:31.005064Z","iopub.execute_input":"2024-11-29T07:17:31.005494Z","iopub.status.idle":"2024-11-29T07:22:53.347356Z","shell.execute_reply.started":"2024-11-29T07:17:31.005456Z","shell.execute_reply":"2024-11-29T07:22:53.346500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Shape of resampled data:\", X_sample_smote.shape)\nprint(\"First 5 resampled labels:\", y_sample_smote[:5])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:22:53.348536Z","iopub.execute_input":"2024-11-29T07:22:53.348808Z","iopub.status.idle":"2024-11-29T07:22:53.353989Z","shell.execute_reply.started":"2024-11-29T07:22:53.348781Z","shell.execute_reply":"2024-11-29T07:22:53.353163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Split the balanced subset into training and validation sets \nfrom sklearn.model_selection import train_test_split\n\n# Split data\nX_train_split, X_val, y_train_split, y_val = train_test_split(\n    X_sample_smote, y_sample_smote, test_size=0.2, random_state=42\n)\n\nprint(\"Training set shape:\", X_train_split.shape)\nprint(\"Validation set shape:\", X_val.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:22:53.355074Z","iopub.execute_input":"2024-11-29T07:22:53.355365Z","iopub.status.idle":"2024-11-29T07:22:53.463219Z","shell.execute_reply.started":"2024-11-29T07:22:53.355339Z","shell.execute_reply":"2024-11-29T07:22:53.462363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Train the baseline logistic regression model\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, f1_score\n\n# Train Logistic Regression\nmodel = LogisticRegression(max_iter=1000, random_state=42)\nmodel.fit(X_train_split, y_train_split)\n\n# Predict on validation set\ny_val_pred = model.predict(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:22:53.464426Z","iopub.execute_input":"2024-11-29T07:22:53.465150Z","iopub.status.idle":"2024-11-29T07:24:08.179439Z","shell.execute_reply.started":"2024-11-29T07:22:53.465088Z","shell.execute_reply":"2024-11-29T07:24:08.178702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(classification_report(y_val, y_val_pred))\nprint(\"Macro F1 Score:\", f1_score(y_val, y_val_pred, average='macro'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:24:08.182766Z","iopub.execute_input":"2024-11-29T07:24:08.183018Z","iopub.status.idle":"2024-11-29T07:24:15.738742Z","shell.execute_reply.started":"2024-11-29T07:24:08.182994Z","shell.execute_reply":"2024-11-29T07:24:15.737760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#optimize parameters to capture more meaningful features\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(max_features=2000, ngram_range=(1, 2), max_df=0.9, min_df=5)\nX_train = vectorizer.fit_transform(train_data_balanced['cleaned_text'])\nX_val = vectorizer.transform(test_data['cleaned_text'])\n\nprint(\"TF-IDF matrix shape:\", X_train.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:24:15.739979Z","iopub.execute_input":"2024-11-29T07:24:15.740296Z","iopub.status.idle":"2024-11-29T07:25:16.196010Z","shell.execute_reply.started":"2024-11-29T07:24:15.740264Z","shell.execute_reply":"2024-11-29T07:25:16.195065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Logistic regression with class weights\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, f1_score\nfrom sklearn.utils.class_weight import compute_class_weight\n\nclasses = train_data_balanced['emotion'].unique()\nclass_weights = compute_class_weight('balanced', classes=classes, y=train_data_balanced['emotion'])\nclass_weights_dict = dict(zip(classes, class_weights))\nprint(\"Class Weights:\", class_weights_dict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:25:16.196858Z","iopub.execute_input":"2024-11-29T07:25:16.197094Z","iopub.status.idle":"2024-11-29T07:25:17.070598Z","shell.execute_reply.started":"2024-11-29T07:25:16.197070Z","shell.execute_reply":"2024-11-29T07:25:17.069644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split training and validation sets\nfrom sklearn.model_selection import train_test_split\nX_train_split, X_val_split, y_train_split, y_val_split = train_test_split(X_train, train_data_balanced['emotion'], test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:25:17.071842Z","iopub.execute_input":"2024-11-29T07:25:17.072088Z","iopub.status.idle":"2024-11-29T07:25:17.732677Z","shell.execute_reply.started":"2024-11-29T07:25:17.072065Z","shell.execute_reply":"2024-11-29T07:25:17.731726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train Logistic Regression\nmodel = LogisticRegression(max_iter=1000, class_weight=class_weights_dict, random_state=42)\nmodel.fit(X_train_split, y_train_split)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:55:30.841528Z","iopub.execute_input":"2024-11-29T07:55:30.841905Z","iopub.status.idle":"2024-11-29T08:10:30.194163Z","shell.execute_reply.started":"2024-11-29T07:55:30.841871Z","shell.execute_reply":"2024-11-29T08:10:30.192104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_val_pred = model.predict(X_val_split)\nprint(classification_report(y_val_split, y_val_pred))\nprint(\"Macro F1 Score:\", f1_score(y_val_split, y_val_pred, average='macro'))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data['cleaned_text'] = test_data['text'].apply(clean_text)\n\n# Transform test data into feature matrix using the same TF-IDF vectorizer\nX_test = vectorizer.transform(test_data['cleaned_text'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:52:10.388951Z","iopub.execute_input":"2024-11-29T07:52:10.389745Z","iopub.status.idle":"2024-11-29T07:52:19.651207Z","shell.execute_reply.started":"2024-11-29T07:52:10.389707Z","shell.execute_reply":"2024-11-29T07:52:19.650246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict emotions for the test set\ny_test_pred = model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T07:52:23.793013Z","iopub.execute_input":"2024-11-29T07:52:23.793392Z","iopub.status.idle":"2024-11-29T07:52:23.858483Z","shell.execute_reply.started":"2024-11-29T07:52:23.793360Z","shell.execute_reply":"2024-11-29T07:52:23.857343Z"}},"outputs":[],"execution_count":null}]}