{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This part is worth 30% of your grade. A report of your work developping the model for the competition (You can use code and comment it). This report should include what your preprocessing steps, the feature engineering steps and an explanation of your model. You can also mention different things you tried and insights you gained.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-23T17:12:24.446342Z","iopub.execute_input":"2022-04-23T17:12:24.446927Z","iopub.status.idle":"2022-04-23T17:12:25.269604Z","shell.execute_reply.started":"2022-04-23T17:12:24.446900Z","shell.execute_reply":"2022-04-23T17:12:25.269154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nfrom pandas.io.json import json_normalize","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:25.270578Z","iopub.execute_input":"2022-04-23T17:12:25.271106Z","iopub.status.idle":"2022-04-23T17:12:25.275340Z","shell.execute_reply.started":"2022-04-23T17:12:25.271082Z","shell.execute_reply":"2022-04-23T17:12:25.274271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation\n## 1.1 Load data","metadata":{}},{"cell_type":"code","source":"# read files \ndata_identification = pd.read_csv(\"../input/dm2021-lab2-hw2/data_identification.csv\")\nemotion = pd.read_csv(\"../input/dm2021-lab2-hw2/emotion.csv\")\nsampleSubmission = pd.read_csv(\"../input/dm2021-lab2-hw2/sampleSubmission.csv\")\nraw_data = pd.read_json(\"../input/dm2021-lab2-hw2/tweets_DM.json\", lines=True,orient='columns')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:25.276261Z","iopub.execute_input":"2022-04-23T17:12:25.276486Z","iopub.status.idle":"2022-04-23T17:12:54.760540Z","shell.execute_reply.started":"2022-04-23T17:12:25.276466Z","shell.execute_reply":"2022-04-23T17:12:54.759842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the identification data\ndata_identification[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:54.762039Z","iopub.execute_input":"2022-04-23T17:12:54.762208Z","iopub.status.idle":"2022-04-23T17:12:54.776867Z","shell.execute_reply.started":"2022-04-23T17:12:54.762187Z","shell.execute_reply":"2022-04-23T17:12:54.775857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check emotion data\nemotion[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:54.778485Z","iopub.execute_input":"2022-04-23T17:12:54.778733Z","iopub.status.idle":"2022-04-23T17:12:54.788383Z","shell.execute_reply.started":"2022-04-23T17:12:54.778699Z","shell.execute_reply":"2022-04-23T17:12:54.787522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the raw data        \nraw_data[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:54.789629Z","iopub.execute_input":"2022-04-23T17:12:54.789857Z","iopub.status.idle":"2022-04-23T17:12:54.820681Z","shell.execute_reply.started":"2022-04-23T17:12:54.789830Z","shell.execute_reply":"2022-04-23T17:12:54.819891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the _source columns\ndf = pd.json_normalize(data=raw_data['_source'])\n\n# rename column names of source\ndf=df.rename(index=str,columns={\"tweet.text\":\"text\", \"tweet.tweet_id\":\"tweet_id\",\"tweet.hashtags\":\"hashtags\"})","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:12:54.822169Z","iopub.execute_input":"2022-04-23T17:12:54.822359Z","iopub.status.idle":"2022-04-23T17:13:12.113402Z","shell.execute_reply.started":"2022-04-23T17:12:54.822333Z","shell.execute_reply":"2022-04-23T17:13:12.112666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check normalized and renamed raw data\ndf[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:12.114399Z","iopub.execute_input":"2022-04-23T17:13:12.114669Z","iopub.status.idle":"2022-04-23T17:13:12.220296Z","shell.execute_reply.started":"2022-04-23T17:13:12.114601Z","shell.execute_reply":"2022-04-23T17:13:12.219514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add identification the dataframe\ndf=pd.merge(df,data_identification, on=\"tweet_id\")","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:12.221935Z","iopub.execute_input":"2022-04-23T17:13:12.222167Z","iopub.status.idle":"2022-04-23T17:13:16.215826Z","shell.execute_reply.started":"2022-04-23T17:13:12.222137Z","shell.execute_reply":"2022-04-23T17:13:16.214889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Clean the text column","metadata":{}},{"cell_type":"code","source":"# clean the text\nimport re\nfrom string import punctuation\n\ndef preprocess_text(text):\n    text = text.lower()  #  lowercase text\n    text = re.sub(f\"[{re.escape(punctuation)}]\", \"\", text)  # remove punctuation\n    text = \" \".join(text.split())  # remove extra spaces, tabs, and new lines\n    return text\n\ndf['text'] = df['text'].map(preprocess_text)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:16.218122Z","iopub.execute_input":"2022-04-23T17:13:16.218659Z","iopub.status.idle":"2022-04-23T17:13:32.496112Z","shell.execute_reply.started":"2022-04-23T17:13:16.218628Z","shell.execute_reply":"2022-04-23T17:13:32.495244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the clean text\ndf[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:32.497150Z","iopub.execute_input":"2022-04-23T17:13:32.497338Z","iopub.status.idle":"2022-04-23T17:13:32.507769Z","shell.execute_reply.started":"2022-04-23T17:13:32.497315Z","shell.execute_reply":"2022-04-23T17:13:32.507399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 Train and test data splitting","metadata":{}},{"cell_type":"code","source":"# split into train and test dataset\ntrain_df = df[df['identification']=='train']\ntrain_df = pd.merge(train_df, emotion, on='tweet_id')\n\ntest_df = df[df['identification']=='test']\ntest_df[\"emotion\"]=\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:32.508533Z","iopub.execute_input":"2022-04-23T17:13:32.508791Z","iopub.status.idle":"2022-04-23T17:13:35.956695Z","shell.execute_reply.started":"2022-04-23T17:13:32.508771Z","shell.execute_reply":"2022-04-23T17:13:35.955894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop the identification and hashtags column\ntrain_df = train_df.drop(['identification'], axis=1)\ntest_df = test_df.drop(['identification'], axis=1)\ntrain_df = train_df.drop(['hashtags'], axis=1)\ntest_df = test_df.drop(['hashtags'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:35.957731Z","iopub.execute_input":"2022-04-23T17:13:35.957949Z","iopub.status.idle":"2022-04-23T17:13:36.853701Z","shell.execute_reply.started":"2022-04-23T17:13:35.957923Z","shell.execute_reply":"2022-04-23T17:13:36.852956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:36.854831Z","iopub.execute_input":"2022-04-23T17:13:36.854993Z","iopub.status.idle":"2022-04-23T17:13:36.864281Z","shell.execute_reply.started":"2022-04-23T17:13:36.854971Z","shell.execute_reply":"2022-04-23T17:13:36.863539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[:10]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:36.865498Z","iopub.execute_input":"2022-04-23T17:13:36.865860Z","iopub.status.idle":"2022-04-23T17:13:36.881754Z","shell.execute_reply.started":"2022-04-23T17:13:36.865830Z","shell.execute_reply":"2022-04-23T17:13:36.881217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.4 Save data in Pickle","metadata":{}},{"cell_type":"code","source":"# save to pickle file, for speed\ntrain_df.to_pickle(\"train_df.pkl\") \ntest_df.to_pickle(\"test_df.pkl\")\n\n# load a pickle file\ntrain_df = pd.read_pickle(\"train_df.pkl\")\ntest_df = pd.read_pickle(\"test_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:36.882599Z","iopub.execute_input":"2022-04-23T17:13:36.882875Z","iopub.status.idle":"2022-04-23T17:13:39.791226Z","shell.execute_reply.started":"2022-04-23T17:13:36.882853Z","shell.execute_reply":"2022-04-23T17:13:39.790762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.5 Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"code","source":"#group to find distribution\ntrain_df.groupby(['emotion']).count()['text']","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:39.792038Z","iopub.execute_input":"2022-04-23T17:13:39.792475Z","iopub.status.idle":"2022-04-23T17:13:40.245796Z","shell.execute_reply.started":"2022-04-23T17:13:39.792449Z","shell.execute_reply":"2022-04-23T17:13:40.244788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the histogram of the data\nlabels = train_df['emotion'].unique()\npost_total = len(train_df)\ndf1 = train_df.groupby(['emotion']).count()['text']\ndf1 = df1.apply(lambda x: round(x*100/post_total,3))\n\n#plot\nfig, ax = plt.subplots(figsize=(10, 5))\nplt.bar(df1.index,df1.values)\n\n#arrange\nplt.ylabel('% of instances')\nplt.xlabel('Emotion')\nplt.title('Emotion distribution')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:40.247458Z","iopub.execute_input":"2022-04-23T17:13:40.247782Z","iopub.status.idle":"2022-04-23T17:13:40.999349Z","shell.execute_reply.started":"2022-04-23T17:13:40.247721Z","shell.execute_reply":"2022-04-23T17:13:40.998205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Feature Engineering\n## 2.1 TF-IDF Vectorizer & Tweet Tokenizer","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import TweetTokenizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n# create a function for the tweet tokenizer from NLTK\ndef tknzr(text):\n    tt = TweetTokenizer()\n    return tt.tokenize(text)\n\nvectorizer = TfidfVectorizer(min_df=20, max_df=0.95, ngram_range=(1,1), stop_words='english', tokenizer=tknzr).fit(train_df['text'])\ntfidf_transformed = vectorizer.transform(train_df['text'])","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:13:41.000654Z","iopub.execute_input":"2022-04-23T17:13:41.000895Z","iopub.status.idle":"2022-04-23T17:16:19.595826Z","shell.execute_reply.started":"2022-04-23T17:13:41.000865Z","shell.execute_reply":"2022-04-23T17:16:19.595071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Model","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(tfidf_transformed, train_df['emotion'], test_size=0.2, random_state=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:16:19.597579Z","iopub.execute_input":"2022-04-23T17:16:19.597774Z","iopub.status.idle":"2022-04-23T17:16:20.036960Z","shell.execute_reply.started":"2022-04-23T17:16:19.597753Z","shell.execute_reply":"2022-04-23T17:16:20.035975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a look at data dimension\nprint('X_train.shape: ', X_train.shape)\nprint('y_train.shape: ', y_train.shape)\nprint('X_test.shape: ', X_test.shape)\nprint('y_test.shape: ', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:16:20.038099Z","iopub.execute_input":"2022-04-23T17:16:20.038283Z","iopub.status.idle":"2022-04-23T17:16:20.045665Z","shell.execute_reply.started":"2022-04-23T17:16:20.038259Z","shell.execute_reply":"2022-04-23T17:16:20.044586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:16:20.046974Z","iopub.execute_input":"2022-04-23T17:16:20.047188Z","iopub.status.idle":"2022-04-23T17:16:20.056357Z","shell.execute_reply.started":"2022-04-23T17:16:20.047161Z","shell.execute_reply":"2022-04-23T17:16:20.055855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.1 Logistic Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\n# build Logistic Regression\nLR_model = LogisticRegression(solver='liblinear', C=10, n_jobs=-1, max_iter=500, tol=0.0001, verbose=2)\n\n# training!\nLR_model.fit(X_train,y_train)\n\n# predict!\ny_train_pred = LR_model.predict(X_train)\ny_test_pred = LR_model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:16:52.208929Z","iopub.execute_input":"2022-04-23T17:16:52.209129Z","iopub.status.idle":"2022-04-23T17:24:09.164724Z","shell.execute_reply.started":"2022-04-23T17:16:52.209107Z","shell.execute_reply":"2022-04-23T17:24:09.163675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# accuracy\nfrom sklearn.metrics import accuracy_score\n\nacc_train = accuracy_score(y_true=y_train, y_pred=y_train_pred)\nacc_test = accuracy_score(y_true=y_test, y_pred=y_test_pred)\n\nprint('training accuracy: {}'.format(round(acc_train, 2)))\nprint('testing accuracy: {}'.format(round(acc_test, 2)))","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:09.166395Z","iopub.execute_input":"2022-04-23T17:24:09.166677Z","iopub.status.idle":"2022-04-23T17:24:11.986395Z","shell.execute_reply.started":"2022-04-23T17:24:09.166649Z","shell.execute_reply":"2022-04-23T17:24:11.985708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# precision, recall, f1-score,\nfrom sklearn.metrics import classification_report\n\nprint(classification_report(y_true=y_test, y_pred=y_test_pred))","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:11.987654Z","iopub.execute_input":"2022-04-23T17:24:11.987963Z","iopub.status.idle":"2022-04-23T17:24:33.870187Z","shell.execute_reply.started":"2022-04-23T17:24:11.987929Z","shell.execute_reply":"2022-04-23T17:24:33.868704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check by confusion matrix\nfrom sklearn.metrics import confusion_matrix\n\ncm = confusion_matrix(y_true=y_test, y_pred=y_test_pred) \nprint(cm)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:33.873131Z","iopub.execute_input":"2022-04-23T17:24:33.873344Z","iopub.status.idle":"2022-04-23T17:24:36.298194Z","shell.execute_reply.started":"2022-04-23T17:24:33.873321Z","shell.execute_reply":"2022-04-23T17:24:36.297279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function for visualizing confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport itertools\n\ndef plot_confusion_matrix(cm, classes, title='Confusion matrix',\n                          cmap=sns.cubehelix_palette(as_cmap=True)):\n    \"\"\"\n    This function is modified from: \n    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n    \"\"\"\n    classes.sort()\n    tick_marks = np.arange(len(classes))    \n    \n    fig, ax = plt.subplots(figsize=(10,10))\n    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n    ax.figure.colorbar(im, ax=ax)\n    ax.set(xticks=np.arange(cm.shape[1]),\n           yticks=np.arange(cm.shape[0]),\n           xticklabels = classes,\n           yticklabels = classes,\n           title = title,\n           xlabel = 'True label',\n           ylabel = 'Predicted label')\n\n    fmt = 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment=\"center\", color=\"white\" if cm[i, j] > thresh else \"black\")\n    ylim_top = len(classes) - 0.5\n    plt.ylim([ylim_top, -.5])\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:36.299521Z","iopub.execute_input":"2022-04-23T17:24:36.299711Z","iopub.status.idle":"2022-04-23T17:24:36.313945Z","shell.execute_reply.started":"2022-04-23T17:24:36.299687Z","shell.execute_reply":"2022-04-23T17:24:36.312872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['emotion'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:36.315102Z","iopub.execute_input":"2022-04-23T17:24:36.315301Z","iopub.status.idle":"2022-04-23T17:24:36.432099Z","shell.execute_reply.started":"2022-04-23T17:24:36.315278Z","shell.execute_reply":"2022-04-23T17:24:36.431079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot your confusion matrix\nmy_tags = ['anticipation', 'sadness', 'fear', 'joy', 'anger', 'trust',\n       'disgust', 'surprise']\nplot_confusion_matrix(cm, classes=my_tags, title='Confusion matrix')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:24:36.433149Z","iopub.execute_input":"2022-04-23T17:24:36.433334Z","iopub.status.idle":"2022-04-23T17:24:36.922310Z","shell.execute_reply.started":"2022-04-23T17:24:36.433312Z","shell.execute_reply":"2022-04-23T17:24:36.921487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict sentiment for test data","metadata":{}},{"cell_type":"code","source":"X_train = vectorizer.transform(train_df['text'])\ny_train = train_df['emotion']\nX_test = vectorizer.transform(test_df['text'])","metadata":{"execution":{"iopub.status.busy":"2022-04-23T18:24:50.525697Z","iopub.execute_input":"2022-04-23T18:24:50.525919Z","iopub.status.idle":"2022-04-23T18:26:57.242026Z","shell.execute_reply.started":"2022-04-23T18:24:50.525895Z","shell.execute_reply":"2022-04-23T18:26:57.240966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training!\nLR_model = LR_model.fit(X_train, y_train)\n\n# predict!\nprediction = LR_model.predict(X_test)\n\n# save as csv!\ntest_df['emotion'] = prediction\noutput = test_df[['tweet_id', 'emotion']].copy()\noutput = output.set_axis([\"id\", \"emotion\"], axis=1)\noutput.to_csv(\"tfidf-logistic-regression.csv\", index=False)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T18:26:57.243214Z","iopub.execute_input":"2022-04-23T18:26:57.243435Z","iopub.status.idle":"2022-04-23T18:37:02.008860Z","shell.execute_reply.started":"2022-04-23T18:26:57.243403Z","shell.execute_reply":"2022-04-23T18:37:02.007879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"execution":{"iopub.status.busy":"2022-04-23T18:37:02.010104Z","iopub.execute_input":"2022-04-23T18:37:02.010324Z","iopub.status.idle":"2022-04-23T18:37:02.022726Z","shell.execute_reply.started":"2022-04-23T18:37:02.010296Z","shell.execute_reply":"2022-04-23T18:37:02.021908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}