{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":87232,"databundleVersionId":9912598,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-09T01:56:22.856958Z","iopub.execute_input":"2024-12-09T01:56:22.860421Z","iopub.status.idle":"2024-12-09T01:56:23.789147Z","shell.execute_reply.started":"2024-12-09T01:56:22.860363Z","shell.execute_reply":"2024-12-09T01:56:23.788357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"category = [\"anger\", \"anticipation\", \"disgust\", \"fear\", \"sadness\", \"surprise\", \"trust\", \"joy\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T01:56:23.791105Z","iopub.execute_input":"2024-12-09T01:56:23.791852Z","iopub.status.idle":"2024-12-09T01:56:23.795607Z","shell.execute_reply.started":"2024-12-09T01:56:23.791809Z","shell.execute_reply":"2024-12-09T01:56:23.794602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tweets_dm = pd.read_json(\"/kaggle/input/dm-2024-isa-5810-lab-2-homework/tweets_DM.json\", lines =True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T01:56:23.796563Z","iopub.execute_input":"2024-12-09T01:56:23.796819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"emotion = pd.read_csv(\"/kaggle/input/dm-2024-isa-5810-lab-2-homework/emotion.csv\")\nidentification = pd.read_csv(\"/kaggle/input/dm-2024-isa-5810-lab-2-homework/data_identification.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract the content of the '_source' column into separate columns\ntweets_dm['tweet_id'] = tweets_dm['_source'].apply(lambda x: x['tweet']['tweet_id'])\ntweets_dm['text'] = tweets_dm['_source'].apply(lambda x: x['tweet']['text'])\ntweets_dm['hashtags'] = tweets_dm['_source'].apply(lambda x: x['tweet']['hashtags'])\n\n# Drop the original '_source' column if it's no longer needed\ntweets_dm = tweets_dm.drop(columns=['_source'])\n\n# Display the updated DataFrame\ntweets_dm\n# Remove '<LH>' from the text column\ntweets_dm['text_cleaned'] = tweets_dm['text'].str.replace(r'<LH>', '', regex=True).str.strip()\n\n# Display the DataFrame with the cleaned text column\nprint(tweets_dm[['text', 'text_cleaned']])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"emotion_tweets  = pd.merge(tweets_dm, emotion, on='tweet_id', how='left')\n\n# Display the merged DataFrame\n# Rename the 'text_cleaned' column to 'text'\nemotion_tweets.drop(columns=['text'], inplace=True)\nemotion_tweets.rename(columns={'text_cleaned': 'text'}, inplace=True)\n\n# Drop the original 'text' column\n\nemotion_tweets","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check nan and drop \n\nemotion_tweets = emotion_tweets.dropna()\nemotion_tweets = emotion_tweets.drop(columns=['_type','_index','_crawldate','_score','hashtags'])\n\nemotion_tweets             ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Total required rows\ntotal_rows = 823944\n\n# Calculate weighted counts for each emotion\nemotion_counts = emotion_tweets['emotion'].value_counts()\nemotion_weights = emotion_counts / emotion_counts.sum()\nweighted_counts = (emotion_weights * total_rows).round().astype(int)\n\n# Sample rows for each emotion\nbalanced_datasets = []\nfor emotion, count in weighted_counts.items():\n    emotion_data = emotion_tweets[emotion_tweets['emotion'] == emotion]\n    sampled_data = emotion_data.sample(n=count, replace=(count > len(emotion_data)), random_state=42)\n    balanced_datasets.append(sampled_data)\n\n# Combine and shuffle\nbalanced_df = pd.concat(balanced_datasets, ignore_index=True)\nbalanced_df = balanced_df.sample(frac=1, random_state=42).reset_index(drop=True)\nbalanced_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nnltk.download('stopwords')\nnltk.download('punkt')\n\ndef preprocess_text(text):\n    # Convert to lowercase\n    text = text.lower()\n    \n    # Remove user mentions\n    text = re.sub(r'@\\w+', '', text)\n    \n    # Remove URLs\n    text = re.sub(r'http\\S+|www.\\S+', '', text)\n    \n    # Remove non-alphanumeric characters\n    text = re.sub(r'[^a-zA-Z\\s]', '', text)\n    \n    # Tokenize text\n    tokens = word_tokenize(text)\n    \n    # Remove stopwords\n    stop_words = set(stopwords.words('english'))\n    tokens = [word for word in tokens if word not in stop_words]\n    \n    # Join tokens back into a single string\n    preprocessed_text = ' '.join(tokens)\n    return preprocessed_text\n\n\nbalanced_df['text'] = balanced_df['text'].apply(preprocess_text)\nbalanced_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n# Separate features (X) and target (y)\nX = balanced_df['text']  # Features\ny = balanced_df['emotion']  # Target\ntweet_ids = balanced_df['tweet_id']  # Unique IDs for submission\n\n# Split data into train and test sets\nX_train, X_test, y_train, y_test, train_ids, test_ids = train_test_split(\n    X, y, tweet_ids, test_size=0.5, random_state=42, stratify=y\n)\n\n\n# Output shapes\nprint(\"Training data:\", X_train.shape, y_train.shape)\nprint(\"Testing data:\", X_test.shape, y_test.shape)\n\n# Output sizes\nprint(\"Training set size:\", len(X_train))\nprint(\"Testing set size:\", len(X_test))\n\n# Example: Ensure features and labels are aligned\nassert len(X_train) == len(y_train), \"Mismatched sizes between X_train and y_train\"\nassert len(X_test) == len(y_test), \"Mismatched sizes between X_test and y_test\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(pd.DataFrame(X_train).isnull().sum())\nprint(pd.DataFrame(X_test).isnull().sum())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nX_train = [str(x) if pd.notnull(x) else '' for x in X_train]\nX_test = [str(x) if pd.notnull(x) else '' for x in X_test]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\n# Convert text to TF-IDF features\nvectorizer = TfidfVectorizer(max_features=3000, stop_words='english')\nX_train_tfidf = vectorizer.fit_transform(X_train)  # Fit on train, transform train\nX_test_tfidf = vectorizer.transform(X_test)       # Only transform test\n\n# Convert sparse matrix to dense if needed\nX_train_dense = X_train_tfidf.toarray()\nX_test_dense = X_test_tfidf.toarray()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlabel_encoder = LabelEncoder()\n\n# Fit on the training labels and transform both train and test sets\ny_train_encoded = label_encoder.fit_transform(y_train)\ny_test_encoded = label_encoder.transform(y_test)\n\nprint(\"Classes:\", label_encoder.classes_)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Input\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import EarlyStopping\n\n# Set GPU device before importing TensorFlow\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'  # Use first GPU\n\n# GPU configuration and optimization\ndef configure_gpu():\n    # List available GPUs\n    gpus = tf.config.list_physical_devices('GPU')\n    if gpus:\n        try:\n            # Configure GPU memory growth\n            for gpu in gpus:\n                tf.config.experimental.set_memory_growth(gpu, True)\n            \n            print(f\"Found {len(gpus)} GPU(s). Using GPU for training.\")\n        except RuntimeError as e:\n            print(f\"GPU configuration error: {e}\")\n    else:\n        print(\"No GPU found. Falling back to CPU.\")\n\n# Configure GPU before model creation\nconfigure_gpu()\n\n# Assuming 8 emotion classes\nnum_classes = len(label_encoder.classes_)\n\n# Improved Neural Network\nimproved_model = Sequential([\n    Input(shape=(3000,)),\n    Dense(128, activation='relu'),\n    Dropout(0.5),\n    Dense(64, activation='relu'),\n    Dropout(0.3),\n    Dense(32, activation='relu'),\n    Dense(num_classes, activation='softmax')  # Multi-class classification\n])\n\n# Compile with GPU-optimized optimizer\nimproved_model.compile(\n    optimizer='adam', \n    loss='categorical_crossentropy', \n    metrics=['accuracy']\n)\n\n# Convert labels to categorical\ny_train_categorical = to_categorical(y_train_encoded, num_classes=num_classes)\ny_test_categorical = to_categorical(y_test_encoded, num_classes=num_classes)\n\n# Train with early stopping \nearly_stopping = EarlyStopping(\n    monitor='val_loss', \n    patience=5, \n    restore_best_weights=True\n)\n\n# Train the model\nhistory = improved_model.fit(\n    X_train_dense, y_train_categorical, \n    validation_split=0.2,\n    epochs=3, \n    batch_size=64,  # You can increase batch size if GPU memory allows\n    callbacks=[early_stopping],\n    verbose=1\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#with tf.device('/GPU:0'):\n #   loss, accuracy = improved_model.evaluate(X_test_dense, y_test_categorical, verbose=1)\n#print(\"Test Loss:\", loss)\n#print(\"Test Accuracy:\", accuracy)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train_categorical","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with tf.device('/GPU:0'):\n    y_pred = improved_model.predict(X_test_dense)\n    y_pred_labels = y_pred.argmax(axis=1)  # Get the class index with the highest probability\n\n# Optional: Decode predicted labels\ny_pred_labels = label_encoder.inverse_transform(y_pred_labels)\ny_pred_labels","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_indices = y_test.index\n\n# Retrieve the corresponding tweet_id values from the original dataset\ntweet_ids = balanced_df.loc[y_test_indices, 'tweet_id'] # Align tweet IDs with test set\n\n# Save to a new variable\ntweet_ids","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tweet_ids = pd.Series(tweet_ids)\nids = tweet_ids.tolist()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Step 3: Create a Submission DataFrame\nsubmission = pd.DataFrame({\n    'id': ids,  # Replace test_ids with the actual IDs of the test data\n    'emotion': y_pred_labels  # Use y_pred for numeric labels or y_pred_labels for string labels\n})\n\n# Step 4: Export to CSV\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"Submission file created: submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train model\n#from sklearn.ensemble import RandomForestClassifier\n#model = RandomForestClassifier(random_state=42)\n#model.fit(X_train_dense, y_train_encoded)\n\n# Make predictions\n# predictions = model.predict(y_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}