{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-11T12:09:39.388992Z","iopub.execute_input":"2022-05-11T12:09:39.389490Z","iopub.status.idle":"2022-05-11T12:09:40.565474Z","shell.execute_reply.started":"2022-05-11T12:09:39.389399Z","shell.execute_reply":"2022-05-11T12:09:40.564349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nfrom pandas.io.json import json_normalize","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:09:40.567187Z","iopub.execute_input":"2022-05-11T12:09:40.567422Z","iopub.status.idle":"2022-05-11T12:09:40.572009Z","shell.execute_reply.started":"2022-05-11T12:09:40.567394Z","shell.execute_reply":"2022-05-11T12:09:40.571096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation\n## 1.1 Load data","metadata":{}},{"cell_type":"code","source":"# read files \ndata_identification = pd.read_csv(\"../input/dm2021-lab2-hw2/data_identification.csv\")\nemotion = pd.read_csv(\"../input/dm2021-lab2-hw2/emotion.csv\")\nsampleSubmission = pd.read_csv(\"../input/dm2021-lab2-hw2/sampleSubmission.csv\")\nraw_data = pd.read_json(\"../input/dm2021-lab2-hw2/tweets_DM.json\", lines=True,orient='columns')","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:09:40.573306Z","iopub.execute_input":"2022-05-11T12:09:40.573653Z","iopub.status.idle":"2022-05-11T12:10:20.422164Z","shell.execute_reply.started":"2022-05-11T12:09:40.573536Z","shell.execute_reply":"2022-05-11T12:10:20.420951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the identification data\ndata_identification[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:20.424736Z","iopub.execute_input":"2022-05-11T12:10:20.424985Z","iopub.status.idle":"2022-05-11T12:10:20.448895Z","shell.execute_reply.started":"2022-05-11T12:10:20.424955Z","shell.execute_reply":"2022-05-11T12:10:20.447979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check emotion data\nemotion[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:20.450128Z","iopub.execute_input":"2022-05-11T12:10:20.450350Z","iopub.status.idle":"2022-05-11T12:10:20.460319Z","shell.execute_reply.started":"2022-05-11T12:10:20.450324Z","shell.execute_reply":"2022-05-11T12:10:20.459333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the raw data        \nraw_data[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:20.462030Z","iopub.execute_input":"2022-05-11T12:10:20.462547Z","iopub.status.idle":"2022-05-11T12:10:20.492800Z","shell.execute_reply.started":"2022-05-11T12:10:20.462499Z","shell.execute_reply":"2022-05-11T12:10:20.491746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the _source columns\ndf = pd.json_normalize(data=raw_data['_source'])\n\n# rename column names of source\ndf=df.rename(index=str,columns={\"tweet.text\":\"text\", \"tweet.tweet_id\":\"tweet_id\",\"tweet.hashtags\":\"hashtags\"})","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:20.494421Z","iopub.execute_input":"2022-05-11T12:10:20.494769Z","iopub.status.idle":"2022-05-11T12:10:37.880004Z","shell.execute_reply.started":"2022-05-11T12:10:20.494719Z","shell.execute_reply":"2022-05-11T12:10:37.879155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check normalized and renamed raw data\ndf[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:37.881268Z","iopub.execute_input":"2022-05-11T12:10:37.881495Z","iopub.status.idle":"2022-05-11T12:10:37.969365Z","shell.execute_reply.started":"2022-05-11T12:10:37.881468Z","shell.execute_reply":"2022-05-11T12:10:37.968265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add identification the dataframe\ndf=pd.merge(df,data_identification, on=\"tweet_id\")","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:37.970449Z","iopub.execute_input":"2022-05-11T12:10:37.971290Z","iopub.status.idle":"2022-05-11T12:10:42.310429Z","shell.execute_reply.started":"2022-05-11T12:10:37.971242Z","shell.execute_reply":"2022-05-11T12:10:42.309565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Clean the text column","metadata":{}},{"cell_type":"code","source":"# clean the text\nimport re\nfrom string import punctuation\n\ndef preprocess_text(text):\n    text = text.lower()  #  lowercase text\n    text = re.sub(f\"[{re.escape(punctuation)}]\", \"\", text)  # remove punctuation\n    text = \" \".join(text.split())  # remove extra spaces, tabs, and new lines\n    return text\n\ndf['text'] = df['text'].map(preprocess_text)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:10:42.313273Z","iopub.execute_input":"2022-05-11T12:10:42.313519Z","iopub.status.idle":"2022-05-11T12:11:04.726610Z","shell.execute_reply.started":"2022-05-11T12:10:42.313491Z","shell.execute_reply":"2022-05-11T12:11:04.725753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the clean text\ndf[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:04.727655Z","iopub.execute_input":"2022-05-11T12:11:04.727940Z","iopub.status.idle":"2022-05-11T12:11:04.744401Z","shell.execute_reply.started":"2022-05-11T12:11:04.727910Z","shell.execute_reply":"2022-05-11T12:11:04.743513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 Train and test data splitting","metadata":{}},{"cell_type":"code","source":"# split into train and test dataset\ntrain_df = df[df['identification']=='train']\ntrain_df = pd.merge(train_df, emotion, on='tweet_id')\n\ntest_df = df[df['identification']=='test']\ntest_df[\"emotion\"]=\"\"","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:04.745560Z","iopub.execute_input":"2022-05-11T12:11:04.745818Z","iopub.status.idle":"2022-05-11T12:11:09.040171Z","shell.execute_reply.started":"2022-05-11T12:11:04.745789Z","shell.execute_reply":"2022-05-11T12:11:09.039161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop the identification and hashtags column\ntrain_df = train_df.drop(['identification'], axis=1)\ntest_df = test_df.drop(['identification'], axis=1)\ntrain_df = train_df.drop(['hashtags'], axis=1)\ntest_df = test_df.drop(['hashtags'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:09.041513Z","iopub.execute_input":"2022-05-11T12:11:09.041794Z","iopub.status.idle":"2022-05-11T12:11:10.300780Z","shell.execute_reply.started":"2022-05-11T12:11:09.041765Z","shell.execute_reply":"2022-05-11T12:11:10.299579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:10.302211Z","iopub.execute_input":"2022-05-11T12:11:10.302507Z","iopub.status.idle":"2022-05-11T12:11:10.315960Z","shell.execute_reply.started":"2022-05-11T12:11:10.302445Z","shell.execute_reply":"2022-05-11T12:11:10.315070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:10.317482Z","iopub.execute_input":"2022-05-11T12:11:10.317769Z","iopub.status.idle":"2022-05-11T12:11:10.336760Z","shell.execute_reply.started":"2022-05-11T12:11:10.317736Z","shell.execute_reply":"2022-05-11T12:11:10.335813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.4 Save data in Pickle","metadata":{}},{"cell_type":"code","source":"# save to pickle file, for speed\ntrain_df.to_pickle(\"train_df.pkl\") \ntest_df.to_pickle(\"test_df.pkl\")\n\n# load a pickle file\ntrain_df = pd.read_pickle(\"train_df.pkl\")\ntest_df = pd.read_pickle(\"test_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:10.338783Z","iopub.execute_input":"2022-05-11T12:11:10.339340Z","iopub.status.idle":"2022-05-11T12:11:14.578219Z","shell.execute_reply.started":"2022-05-11T12:11:10.339287Z","shell.execute_reply":"2022-05-11T12:11:14.577192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.5 Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"code","source":"#group to find distribution\ntrain_df.groupby(['emotion']).count()['text']","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:14.579447Z","iopub.execute_input":"2022-05-11T12:11:14.579710Z","iopub.status.idle":"2022-05-11T12:11:15.298704Z","shell.execute_reply.started":"2022-05-11T12:11:14.579678Z","shell.execute_reply":"2022-05-11T12:11:15.297846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the histogram of the data\nlabels = train_df['emotion'].unique()\npost_total = len(train_df)\ndf1 = train_df.groupby(['emotion']).count()['text']\ndf1 = df1.apply(lambda x: round(x*100/post_total,3))\n\n#plot\nfig, ax = plt.subplots(figsize=(10, 5))\nplt.bar(df1.index,df1.values)\n\n#arrange\nplt.ylabel('% of instances')\nplt.xlabel('Emotion')\nplt.title('Emotion distribution')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:15.299848Z","iopub.execute_input":"2022-05-11T12:11:15.300088Z","iopub.status.idle":"2022-05-11T12:11:16.379331Z","shell.execute_reply.started":"2022-05-11T12:11:15.300058Z","shell.execute_reply":"2022-05-11T12:11:16.378538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Count Vectorizer & Tweet Tokenizer","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom nltk.tokenize import TweetTokenizer\n\n# create a function for the tweet tokenizer from NLTK\ndef tknzr(text):\n    tt = TweetTokenizer()\n    return tt.tokenize(text)\n\ncount_vect = CountVectorizer(max_features=100000, tokenizer=tknzr).fit(train_df['text'])\nbow_transformed = count_vect.transform(train_df['text'])","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:11:16.380498Z","iopub.execute_input":"2022-05-11T12:11:16.380894Z","iopub.status.idle":"2022-05-11T12:17:17.218318Z","shell.execute_reply.started":"2022-05-11T12:11:16.380852Z","shell.execute_reply":"2022-05-11T12:17:17.217019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Model","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train = bow_transformed\ny_train = train_df['emotion']\nX_test = count_vect.transform(test_df['text'])\ny_test = test_df['emotion']","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:17:17.221934Z","iopub.execute_input":"2022-05-11T12:17:17.222314Z","iopub.status.idle":"2022-05-11T12:18:16.094462Z","shell.execute_reply.started":"2022-05-11T12:17:17.222265Z","shell.execute_reply":"2022-05-11T12:18:16.093296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a look at data dimension\nprint('X_train.shape: ', X_train.shape)\nprint('y_train.shape: ', y_train.shape)\nprint('X_test.shape: ', X_test.shape)\nprint('y_test.shape: ', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:16.096021Z","iopub.execute_input":"2022-05-11T12:18:16.096259Z","iopub.status.idle":"2022-05-11T12:18:16.103039Z","shell.execute_reply.started":"2022-05-11T12:18:16.096229Z","shell.execute_reply":"2022-05-11T12:18:16.101992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:16.104794Z","iopub.execute_input":"2022-05-11T12:18:16.105101Z","iopub.status.idle":"2022-05-11T12:18:16.115929Z","shell.execute_reply.started":"2022-05-11T12:18:16.105057Z","shell.execute_reply":"2022-05-11T12:18:16.115144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.1 Deal with categorical label","metadata":{}},{"cell_type":"code","source":"# deal with label (string -> one-hot)\n\nimport keras\nfrom keras import utils as np_utils\nfrom sklearn.preprocessing import LabelEncoder\n\nlabel_encoder = LabelEncoder()\nlabel_encoder.fit(y_train)\nprint('check label: ', label_encoder.classes_)\nprint('\\n## Before convert')\nprint('y_train[0:4]:\\n', y_train[0:8])\nprint('\\ny_train.shape: ', y_train.shape)\n\ndef label_encode(le, labels):\n    enc = le.transform(labels)\n    return keras.utils.np_utils.to_categorical(enc)\n\ndef label_decode(le, one_hot_label):\n    dec = np.argmax(one_hot_label, axis=1)\n    return le.inverse_transform(dec)\n\ny_train = label_encode(label_encoder, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:16.117495Z","iopub.execute_input":"2022-05-11T12:18:16.118033Z","iopub.status.idle":"2022-05-11T12:18:24.809062Z","shell.execute_reply.started":"2022-05-11T12:18:16.117987Z","shell.execute_reply":"2022-05-11T12:18:24.808155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Build model","metadata":{}},{"cell_type":"code","source":"# I/O check\ninput_shape = X_train.shape[1]\nprint('input_shape: ', input_shape)\n\noutput_shape = len(label_encoder.classes_)\nprint('output_shape: ', output_shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:24.810541Z","iopub.execute_input":"2022-05-11T12:18:24.810777Z","iopub.status.idle":"2022-05-11T12:18:24.817358Z","shell.execute_reply.started":"2022-05-11T12:18:24.810750Z","shell.execute_reply":"2022-05-11T12:18:24.816506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.layers import ReLU, Softmax\n\n# input layer\nmodel_input = Input(shape=(input_shape, ))  # 500\nX = model_input\n\n# 1st hidden layer\nX_W1 = Dense(units=64)(X)  # 64\nH1 = ReLU()(X_W1)\n\n# 2nd hidden layer\nH1_W2 = Dense(units=64)(H1)  # 64\nH2 = ReLU()(H1_W2)\n\n# output layer\nH2_W3 = Dense(units=output_shape)(H2)  # 4\nH3 = Softmax()(H2_W3)\n\nmodel_output = H3\n\n# create model\nmodel = Model(inputs=[model_input], outputs=[model_output])\n\n# loss function & optimizer\nmodel.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\n# show model construction\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:24.818651Z","iopub.execute_input":"2022-05-11T12:18:24.819161Z","iopub.status.idle":"2022-05-11T12:18:25.058304Z","shell.execute_reply.started":"2022-05-11T12:18:24.819126Z","shell.execute_reply":"2022-05-11T12:18:25.057368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import CSVLogger\n\ncsv_logger = CSVLogger('training_log.csv')\n\n# training setting\nepochs = 10\nbatch_size = 32\n\n# training!\nhistory = model.fit(X_train, y_train, \n                    epochs=epochs, \n                    batch_size=batch_size, \n                    callbacks=[csv_logger])\n\nprint('training finish')","metadata":{"execution":{"iopub.status.busy":"2022-05-11T12:18:25.060333Z","iopub.execute_input":"2022-05-11T12:18:25.060713Z","iopub.status.idle":"2022-05-11T17:42:31.286119Z","shell.execute_reply.started":"2022-05-11T12:18:25.060646Z","shell.execute_reply":"2022-05-11T17:42:31.284302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3 Predict on testing data","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\n# predict!\nprediction = model.predict(X_test, batch_size=128)\nprediction = label_decode(label_encoder, prediction)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T17:42:51.158886Z","iopub.execute_input":"2022-05-11T17:42:51.159202Z","iopub.status.idle":"2022-05-11T17:42:59.824624Z","shell.execute_reply.started":"2022-05-11T17:42:51.159174Z","shell.execute_reply":"2022-05-11T17:42:59.823872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save as csv!\ntest_df['emotion'] = prediction\noutput = test_df[['tweet_id', 'emotion']].copy()\noutput = output.set_axis([\"id\", \"emotion\"], axis=1)\noutput.to_csv(\"bow-neural-network.csv\", index=False)\noutput","metadata":{"execution":{"iopub.status.busy":"2022-05-11T17:42:59.826115Z","iopub.execute_input":"2022-05-11T17:42:59.826479Z","iopub.status.idle":"2022-05-11T17:43:00.827412Z","shell.execute_reply.started":"2022-05-11T17:42:59.826449Z","shell.execute_reply":"2022-05-11T17:43:00.826540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}