{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Lab2-Kaggle","metadata":{}},{"cell_type":"markdown","source":"This notebook is for Lab2 Kaggle competition. For further information, see [kaggle_competition](https://www.kaggle.com/competitions/dm2022-isa5810-lab2-homework/overview).","metadata":{}},{"cell_type":"markdown","source":"Cited from `DM2022-Lab2-Master.ipynb`:","metadata":{}},{"cell_type":"markdown","source":"```{cite}\n5. Other things you can try\n\nThus, there are several things you can try that will affect your results. In order to yield better results, you can experiment by: \n- Trying different features (Feature engineering)e.g Word2Vec, PCA, LDA, FastText, Clustering\n- Trying different models\n- Analyzing your results and interpret them to improve your feature engineering/model building process\n- Iterate through the steps above until finding a satisfying result\n\nRemember that you should also consider the task at hand and the model you'll feed the data to. \n```","metadata":{}},{"cell_type":"markdown","source":"## 1. Data preparation","metadata":{}},{"cell_type":"markdown","source":"### 1.1 Load Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:52:10.287189Z","iopub.execute_input":"2022-11-21T11:52:10.287639Z","iopub.status.idle":"2022-11-21T11:52:10.31151Z","shell.execute_reply.started":"2022-11-21T11:52:10.287552Z","shell.execute_reply":"2022-11-21T11:52:10.310354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### load csv data\ndata_id = pd.read_csv(\"../input/dm2022-isa5810-lab2-homework/data_identification.csv\") #, index_col='tweet_id')\nemotion = pd.read_csv(\"../input/dm2022-isa5810-lab2-homework/emotion.csv\")#, index_col='tweet_id')","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:52:53.330602Z","iopub.execute_input":"2022-11-21T11:52:53.331406Z","iopub.status.idle":"2022-11-21T11:52:56.492255Z","shell.execute_reply.started":"2022-11-21T11:52:53.331364Z","shell.execute_reply":"2022-11-21T11:52:56.491196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### load json data\ntweets = pd.read_json(\"../input/dm2022-isa5810-lab2-homework/tweets_DM.json\", lines=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:53:03.828724Z","iopub.execute_input":"2022-11-21T11:53:03.829171Z","iopub.status.idle":"2022-11-21T11:53:35.060487Z","shell.execute_reply.started":"2022-11-21T11:53:03.829123Z","shell.execute_reply":"2022-11-21T11:53:35.059301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data_id.shape)\ndata_id.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:40:18.756333Z","iopub.execute_input":"2022-11-21T07:40:18.756754Z","iopub.status.idle":"2022-11-21T07:40:18.770354Z","shell.execute_reply.started":"2022-11-21T07:40:18.756719Z","shell.execute_reply":"2022-11-21T07:40:18.768799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(emotion.shape)\nemotion.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:40:33.419781Z","iopub.execute_input":"2022-11-21T07:40:33.420214Z","iopub.status.idle":"2022-11-21T07:40:33.432461Z","shell.execute_reply.started":"2022-11-21T07:40:33.420178Z","shell.execute_reply":"2022-11-21T07:40:33.431017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweets.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(tweets['_source'])","metadata":{"execution":{"iopub.status.busy":"2022-11-19T07:45:42.299822Z","iopub.execute_input":"2022-11-19T07:45:42.300189Z","iopub.status.idle":"2022-11-19T07:45:42.307007Z","shell.execute_reply.started":"2022-11-19T07:45:42.300158Z","shell.execute_reply":"2022-11-19T07:45:42.305812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweets['_source'][0]['tweet']","metadata":{"execution":{"iopub.status.busy":"2022-11-19T07:45:45.454805Z","iopub.execute_input":"2022-11-19T07:45:45.455171Z","iopub.status.idle":"2022-11-19T07:45:45.462682Z","shell.execute_reply.started":"2022-11-19T07:45:45.455139Z","shell.execute_reply":"2022-11-19T07:45:45.461655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweet_id = [tweets['_source'][i]['tweet']['tweet_id'] for i in range(len(tweets['_source']))]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:53:52.779031Z","iopub.execute_input":"2022-11-21T11:53:52.77949Z","iopub.status.idle":"2022-11-21T11:54:05.371022Z","shell.execute_reply.started":"2022-11-21T11:53:52.779454Z","shell.execute_reply":"2022-11-21T11:54:05.369673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweet_text = [tweets['_source'][i]['tweet']['text'] for i in range(len(tweets['_source']))]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:54:17.858347Z","iopub.execute_input":"2022-11-21T11:54:17.858711Z","iopub.status.idle":"2022-11-21T11:54:30.557019Z","shell.execute_reply.started":"2022-11-21T11:54:17.858679Z","shell.execute_reply":"2022-11-21T11:54:30.555694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweet = pd.DataFrame({'tweet_id': tweet_id, 'text': tweet_text})\n# tweet = pd.DataFrame({'text' : tweet_text}, index=tweet_id)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:54:36.290554Z","iopub.execute_input":"2022-11-21T11:54:36.290975Z","iopub.status.idle":"2022-11-21T11:54:36.960958Z","shell.execute_reply.started":"2022-11-21T11:54:36.290942Z","shell.execute_reply":"2022-11-21T11:54:36.959875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tweet.shape)\ntweet.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:40:52.11136Z","iopub.execute_input":"2022-11-21T07:40:52.111742Z","iopub.status.idle":"2022-11-21T07:40:52.127079Z","shell.execute_reply.started":"2022-11-21T07:40:52.111711Z","shell.execute_reply":"2022-11-21T07:40:52.125889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### concatenate dfs to one df by tweet_id\ndf = data_id.merge(emotion, on='tweet_id', how='outer').merge(tweet, on='tweet_id', how='outer')\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:54:51.138455Z","iopub.execute_input":"2022-11-21T11:54:51.139552Z","iopub.status.idle":"2022-11-21T11:54:57.657779Z","shell.execute_reply.started":"2022-11-21T11:54:51.139498Z","shell.execute_reply":"2022-11-21T11:54:57.656648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:48:15.693891Z","iopub.execute_input":"2022-11-21T07:48:15.694353Z","iopub.status.idle":"2022-11-21T07:48:15.708195Z","shell.execute_reply.started":"2022-11-21T07:48:15.694312Z","shell.execute_reply":"2022-11-21T07:48:15.706838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### train-test split\ntrain_df = df[df['identification']=='train']\ntest_df = df[df['identification']=='test']","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:55:14.590964Z","iopub.execute_input":"2022-11-21T11:55:14.591403Z","iopub.status.idle":"2022-11-21T11:55:15.198608Z","shell.execute_reply.started":"2022-11-21T11:55:14.591368Z","shell.execute_reply":"2022-11-21T11:55:15.197706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_df.emotion) - sum(test_df.emotion.isnull())","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:49:55.809053Z","iopub.execute_input":"2022-11-21T07:49:55.809429Z","iopub.status.idle":"2022-11-21T07:49:55.872201Z","shell.execute_reply.started":"2022-11-21T07:49:55.809399Z","shell.execute_reply":"2022-11-21T07:49:55.871119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"emotion in test dataset is empty.","metadata":{}},{"cell_type":"markdown","source":"### 1.2 Save Data (to .pkl)","metadata":{}},{"cell_type":"code","source":"## save to pickle file\ntrain_df.to_pickle(\"./train_df.pkl\") \ntest_df.to_pickle(\"./test_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:55:24.716598Z","iopub.execute_input":"2022-11-21T11:55:24.717014Z","iopub.status.idle":"2022-11-21T11:55:27.411176Z","shell.execute_reply.started":"2022-11-21T11:55:24.716979Z","shell.execute_reply":"2022-11-21T11:55:27.409894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n## load a pickle file\ntrain_df = pd.read_pickle(\"./train_df.pkl\")\ntest_df = pd.read_pickle(\"./test_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:55:47.258726Z","iopub.execute_input":"2022-11-21T11:55:47.259127Z","iopub.status.idle":"2022-11-21T11:55:48.65031Z","shell.execute_reply.started":"2022-11-21T11:55:47.259098Z","shell.execute_reply":"2022-11-21T11:55:48.649358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.drop(columns=['identification'], axis=1)\n# test_df.drop(columns=['identification'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)\nprint(test_df.shape[0]/(test_df.shape[0]+train_df.shape[0]))","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:56:10.862531Z","iopub.execute_input":"2022-11-21T11:56:10.86299Z","iopub.status.idle":"2022-11-21T11:56:10.869386Z","shell.execute_reply.started":"2022-11-21T11:56:10.862953Z","shell.execute_reply":"2022-11-21T11:56:10.86842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Exploratory data analysis (EDA)","metadata":{}},{"cell_type":"code","source":"# group to find distribution\ntrain_df.groupby(['emotion']).count()['text']","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:52:57.674376Z","iopub.execute_input":"2022-11-21T07:52:57.674764Z","iopub.status.idle":"2022-11-21T07:52:58.343204Z","shell.execute_reply.started":"2022-11-21T07:52:57.674732Z","shell.execute_reply":"2022-11-21T07:52:58.341969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# the histogram of the data\nlabels = train_df['emotion'].unique()\npost_total = len(train_df)\ndf1 = train_df.groupby(['emotion']).count()['text']\ndf1 = df1.apply(lambda x: round(x*100/post_total,3))\n\n#plot\nfig, ax = plt.subplots(figsize=(5,3))\nplt.bar(df1.index,df1.values)\n\n#arrange\nplt.ylabel('% of instances')\nplt.xlabel('Emotion')\nplt.xticks(rotation=90)\nplt.title('Emotion distribution')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:53:01.739742Z","iopub.execute_input":"2022-11-21T07:53:01.740512Z","iopub.status.idle":"2022-11-21T07:53:02.733822Z","shell.execute_reply.started":"2022-11-21T07:53:01.74047Z","shell.execute_reply":"2022-11-21T07:53:02.732764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### * prepare data, split train_df to training set and validation set","metadata":{}},{"cell_type":"code","source":"# shuffle dataset\ntrain_df = train_df.sample(frac=1)\ntest_df = test_df.sample(frac=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:56:21.0724Z","iopub.execute_input":"2022-11-21T11:56:21.072798Z","iopub.status.idle":"2022-11-21T11:56:22.134609Z","shell.execute_reply.started":"2022-11-21T11:56:21.072766Z","shell.execute_reply":"2022-11-21T11:56:22.133475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train.shape: ', train_df.shape)\nprint('test.shape: ', test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:56:25.723293Z","iopub.execute_input":"2022-11-21T11:56:25.723703Z","iopub.status.idle":"2022-11-21T11:56:25.72968Z","shell.execute_reply.started":"2022-11-21T11:56:25.723672Z","shell.execute_reply":"2022-11-21T11:56:25.728555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# the histogram of the data\nlabels = train_df['emotion'].unique()\npost_total = len(train_df)\ndf2 = train_df.groupby(['emotion']).count()['text']\ndf2 = df2.apply(lambda x: round(x*100/post_total,3))\n\n#plot\nfig, ax = plt.subplots(figsize=(5,3))\nplt.bar(df2.index,df2.values)\n\n#arrange\nplt.ylabel('% of instances')\nplt.xlabel('Emotion')\nplt.xticks(rotation=90)\nplt.title('Emotion sample distribution')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T07:53:43.879774Z","iopub.execute_input":"2022-11-21T07:53:43.88048Z","iopub.status.idle":"2022-11-21T07:53:44.119651Z","shell.execute_reply.started":"2022-11-21T07:53:43.880441Z","shell.execute_reply":"2022-11-21T07:53:44.118539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Feature engineering","metadata":{}},{"cell_type":"markdown","source":"CouterVector, tfidfVector, W2V, PCA, LDA,FastText, Clustering","metadata":{}},{"cell_type":"markdown","source":"### 2.1 CounterVector","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nimport nltk\n# nltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:56:30.792052Z","iopub.execute_input":"2022-11-21T11:56:30.792505Z","iopub.status.idle":"2022-11-21T11:56:31.966849Z","shell.execute_reply.started":"2022-11-21T11:56:30.792469Z","shell.execute_reply":"2022-11-21T11:56:31.965596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# build analyzers (bag-of-words)\nBOW500_vectorizer = CountVectorizer(max_features=500, tokenizer=nltk.word_tokenize) \n\n# 1. Learn a vocabulary dictionary of all tokens in the raw documents.\n# apply analyzer to training data\nBOW500_vectorizer.fit(train_df['text'])\n\n# 2. Transform documents to document-term matrix.\ntrain_data_BOW_features_500 = BOW500_vectorizer.transform(train_df['text'])\n\n## check dimensio\ntrain_data_BOW_features_500.shape\n\n# observe some feature names\nBOW500_vectorizer.get_feature_names_out()[100:110]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T11:56:40.030859Z","iopub.execute_input":"2022-11-21T11:56:40.031321Z","iopub.status.idle":"2022-11-21T12:09:27.994461Z","shell.execute_reply.started":"2022-11-21T11:56:40.031287Z","shell.execute_reply":"2022-11-21T12:09:27.993233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 TFIDFVector","metadata":{}},{"cell_type":"code","source":"# import module\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# build vectorizer (tfidf)\nTFIDF500_vectorizer = TfidfVectorizer(max_features=500, tokenizer=nltk.word_tokenize) \n\n# apply analyzer to training data\nTFIDF500_vectorizer.fit(train_sample['text'])\n\ntrain_data_TFIDF_features_500 = TFIDF500_vectorizer.transform(train_sample['text'])\n\n## check features\nTFIDF500_vectorizer.get_feature_names_out()[100:110]","metadata":{"execution":{"iopub.status.busy":"2022-11-19T10:00:32.04398Z","iopub.execute_input":"2022-11-19T10:00:32.044342Z","iopub.status.idle":"2022-11-19T10:00:36.457346Z","shell.execute_reply.started":"2022-11-19T10:00:32.044313Z","shell.execute_reply":"2022-11-19T10:00:36.455843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nBOW_PCA = PCA(n_components = 8).fit_transform(train_data_BOW_features_500.toarray())","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:37:54.718604Z","iopub.execute_input":"2022-11-19T12:37:54.719042Z","iopub.status.idle":"2022-11-19T12:37:55.286372Z","shell.execute_reply.started":"2022-11-19T12:37:54.718987Z","shell.execute_reply":"2022-11-19T12:37:55.285028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories = list(train_df.groupby(['emotion']).groups.keys())","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:32:12.89992Z","iopub.execute_input":"2022-11-19T12:32:12.900928Z","iopub.status.idle":"2022-11-19T12:32:13.195777Z","shell.execute_reply.started":"2022-11-19T12:32:12.900882Z","shell.execute_reply":"2022-11-19T12:32:13.19465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:38:00.516747Z","iopub.execute_input":"2022-11-19T12:38:00.517176Z","iopub.status.idle":"2022-11-19T12:38:00.522263Z","shell.execute_reply.started":"2022-11-19T12:38:00.517137Z","shell.execute_reply":"2022-11-19T12:38:00.521096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"color = ['crimson', 'deepskyblue', 'limegreen', 'darkviolet', 'yellow', 'mediumblue', 'gold', 'greenyellow']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot\nfig = plt.figure(figsize = (25,10))\nax = fig.subplots()\n\nfor c, category in zip(col, categories):\n    xs = BOW_PCA[train_df['emotion'] == category].T[0]\n    ys = BOW_PCA[train_df['emotion'] == category].T[1]\n   \n    ax.scatter(xs, ys, c = c, marker='o')\n\nax.grid(color='gray', linestyle=':', linewidth=2, alpha=0.2)\nax.set_xlabel('\\nX Label')\nax.set_ylabel('\\nY Label')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:38:06.891231Z","iopub.execute_input":"2022-11-19T12:38:06.891628Z","iopub.status.idle":"2022-11-19T12:38:07.289247Z","shell.execute_reply.started":"2022-11-19T12:38:06.891594Z","shell.execute_reply":"2022-11-19T12:38:07.288088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"it doesn't look right?","metadata":{}},{"cell_type":"markdown","source":"### 2.4 LDA","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import LatentDirichletAllocation","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:46:07.80159Z","iopub.execute_input":"2022-11-19T12:46:07.802025Z","iopub.status.idle":"2022-11-19T12:46:07.807522Z","shell.execute_reply.started":"2022-11-19T12:46:07.801977Z","shell.execute_reply":"2022-11-19T12:46:07.806268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LDA = LatentDirichletAllocation(n_components=8, random_state=42)\nBOW_LDA = LDA.fit_transform(train_data_BOW_features_500.toarray())","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:48:51.410298Z","iopub.execute_input":"2022-11-19T12:48:51.411205Z","iopub.status.idle":"2022-11-19T12:49:28.925791Z","shell.execute_reply.started":"2022-11-19T12:48:51.411165Z","shell.execute_reply":"2022-11-19T12:49:28.924256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Word2Vect","metadata":{}},{"cell_type":"code","source":"from gensim.models import Word2Vec\n\n## setting\nvector_dim = 100\nwindow_size = 5\nmin_count = 1\ntraining_epochs = 20\n\n## model\nword2vec_model = Word2Vec(sentences=training_corpus, \n                          vector_size=vector_dim, window=window_size, \n                          min_count=min_count, epochs=training_epochs)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import KeyedVectors\n\n## Note: this model is very huge, this will take some time ...\nmodel_path = \"../input/googlenews/GoogleNews-vectors-negative300.bin\"\nw2v_google_model = KeyedVectors.load_word2vec_format(model_path, binary=True)\nprint('load ok')\n\nw2v_google_model.most_similar('happy', topn=10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.6 Glove","metadata":{}},{"cell_type":"code","source":"import gensim.downloader as api\n\n## If you see `SSL: CERTIFICATE_VERIFY_FAILED` error, use this:\nimport ssl\nimport urllib.request\nssl._create_default_https_context = ssl._create_unverified_context\n\nglove_twitter_25_model = api.load(\"glove-twitter-25\")\nprint('load ok')\n\nglove_twitter_25_model.most_similar('happy', topn=10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.7 FastText","metadata":{}},{"cell_type":"code","source":"from gensim.models import word2vec, fasttext\n\n\n# Settings\nseed = 666\nsg = 0\nwindow_size = 10\nvector_size = 100\nmin_count = 1\nworkers = 8\nepochs = 5\nbatch_words = 10000\n\n\n# Train\n# train_data = word2vec.LineSentence('wiki_text_seg.txt')\nmodel = fasttext.FastText(\n    train_data,\n    min_count=min_count,\n    size=vector_size,\n    workers=workers,\n    iter=epochs,\n    window=window_size,\n    sg=sg,\n    seed=seed,\n    batch_words=batch_words,\n)\n\n\nmodel.save('fasttext.model')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Model & Results Evaluation","metadata":{}},{"cell_type":"code","source":"# for a classificaiton problem, you need to provide both training & testing data\nX_train = BOW500_vectorizer.transform(train_df['text'])\ny_train = train_df['emotion']\n\nX_test = BOW500_vectorizer.transform(test_df['text'])\ny_test = test_df['emotion']# all of this sould be nan\n\n## take a look at data dimension is a good habit  :)\nprint('X_train.shape: ', X_train.shape)\nprint('y_train.shape: ', y_train.shape)\nprint('X_test.shape: ', X_test.shape)\nprint('y_test.shape: ', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T12:17:34.587042Z","iopub.execute_input":"2022-11-21T12:17:34.587498Z","iopub.status.idle":"2022-11-21T12:25:54.661948Z","shell.execute_reply.started":"2022-11-21T12:17:34.587464Z","shell.execute_reply":"2022-11-21T12:25:54.661086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.1 Decision Tree","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\n\n## build DecisionTree model\nDT_model = DecisionTreeClassifier(random_state=1)\n\n## training!\nDT_model = DT_model.fit(X_train, y_train)\n\n## predict!\ny_train_pred = DT_model.predict(X_train)\ny_test_pred = DT_model.predict(X_test)\n\n## so we get the pred result\ny_test_pred[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T12:42:38.667922Z","iopub.execute_input":"2022-11-21T12:42:38.668329Z","iopub.status.idle":"2022-11-21T13:59:11.417057Z","shell.execute_reply.started":"2022-11-21T12:42:38.668296Z","shell.execute_reply":"2022-11-21T13:59:11.415875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-21T09:46:04.710088Z","iopub.execute_input":"2022-11-21T09:46:04.710498Z","iopub.status.idle":"2022-11-21T09:46:04.718756Z","shell.execute_reply.started":"2022-11-21T09:46:04.710465Z","shell.execute_reply":"2022-11-21T09:46:04.717377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['tweet_id'].shape","metadata":{"execution":{"iopub.status.busy":"2022-11-21T09:47:16.591542Z","iopub.execute_input":"2022-11-21T09:47:16.591971Z","iopub.status.idle":"2022-11-21T09:47:16.603613Z","shell.execute_reply.started":"2022-11-21T09:47:16.591921Z","shell.execute_reply":"2022-11-21T09:47:16.602201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"answer = pd.DataFrame({'id': test_df['tweet_id'], 'emotion': y_test_pred})","metadata":{"execution":{"iopub.status.busy":"2022-11-21T14:01:14.428976Z","iopub.execute_input":"2022-11-21T14:01:14.429485Z","iopub.status.idle":"2022-11-21T14:01:14.458694Z","shell.execute_reply.started":"2022-11-21T14:01:14.429446Z","shell.execute_reply":"2022-11-21T14:01:14.457252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"answer.to_csv('submission.csv', index=False) #submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-11-21T14:01:22.943506Z","iopub.execute_input":"2022-11-21T14:01:22.943954Z","iopub.status.idle":"2022-11-21T14:01:23.586088Z","shell.execute_reply.started":"2022-11-21T14:01:22.943918Z","shell.execute_reply":"2022-11-21T14:01:23.584744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('./submission.csv').shape","metadata":{"execution":{"iopub.status.busy":"2022-11-21T14:22:04.152273Z","iopub.execute_input":"2022-11-21T14:22:04.152805Z","iopub.status.idle":"2022-11-21T14:22:04.448076Z","shell.execute_reply.started":"2022-11-21T14:22:04.152756Z","shell.execute_reply":"2022-11-21T14:22:04.446404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## dont know why the file didnt show\n## so i try delete&rewrite\n# import os\n# os.remove(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-21T10:39:08.218051Z","iopub.execute_input":"2022-11-21T10:39:08.219316Z","iopub.status.idle":"2022-11-21T10:39:08.225626Z","shell.execute_reply.started":"2022-11-21T10:39:08.219272Z","shell.execute_reply":"2022-11-21T10:39:08.22426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Naive Baysian","metadata":{}},{"cell_type":"code","source":"# Answer here\nfrom sklearn.naive_bayes import MultinomialNB\n\n## build DecisionTree model\nNB_model = MultinomialNB()\n## training!\nNB_model = NB_model.fit(X_train, y_train)\n\n## predict!\ny_train_pred = NB_model.predict(X_train)\ny_test_pred = NB_model.predict(X_test)\n\n## so we get the pred result\ny_test_pred[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T06:18:53.849071Z","iopub.execute_input":"2022-11-21T06:18:53.849495Z","iopub.status.idle":"2022-11-21T06:18:53.902055Z","shell.execute_reply.started":"2022-11-21T06:18:53.849458Z","shell.execute_reply":"2022-11-21T06:18:53.90087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Deep learning & Result Evaluation","metadata":{}},{"cell_type":"markdown","source":"### 4.1 Prepare data (X,y)","metadata":{}},{"cell_type":"code","source":"import keras","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.2 Prepare categorical label (y)","metadata":{}},{"cell_type":"code","source":"## deal with label (string -> one-hot)\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.utils import np_utils # keras==2.4.0 and tensorflow==2.3.0","metadata":{"execution":{"iopub.status.busy":"2022-11-21T06:22:28.230348Z","iopub.execute_input":"2022-11-21T06:22:28.230761Z","iopub.status.idle":"2022-11-21T06:22:28.237127Z","shell.execute_reply.started":"2022-11-21T06:22:28.230727Z","shell.execute_reply":"2022-11-21T06:22:28.235648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder = LabelEncoder()\nlabel_encoder.fit(y_train)\nprint('check label: ', label_encoder.classes_)\nprint('\\n## Before convert')\nprint('y_train[0:4]:\\n', y_train[0:4])\nprint('\\ny_train.shape: ', y_train.shape)\nprint('y_test.shape: ', y_test.shape)\n\ndef label_encode(le, labels):\n    enc = le.transform(labels)\n    return np_utils.to_categorical(enc)\n\ndef label_decode(le, one_hot_label):\n    dec = np.argmax(one_hot_label, axis=1)\n    return le.inverse_transform(dec)\n\ny_train = label_encode(label_encoder, y_train)\n# y_test = label_encode(label_encoder, y_test)\n\nprint('\\n\\n## After convert')\nprint('y_train[0:4]:\\n', y_train[0:4])\nprint('\\ny_train.shape: ', y_train.shape)\nprint('y_test.shape: ', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T06:30:12.896278Z","iopub.execute_input":"2022-11-21T06:30:12.896682Z","iopub.status.idle":"2022-11-21T06:30:12.913988Z","shell.execute_reply.started":"2022-11-21T06:30:12.896633Z","shell.execute_reply":"2022-11-21T06:30:12.912515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.3 Build model","metadata":{}},{"cell_type":"code","source":"# I/O check\ninput_shape = X_train.shape[1]\nprint('input_shape: ', input_shape)\n\noutput_shape = len(label_encoder.classes_)\nprint('output_shape: ', output_shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.layers import ReLU, Softmax\n\n# input layer\nmodel_input = Input(shape=(input_shape, ))  # 500\nX = model_input\n\n# 1st hidden layer\nX_W1 = Dense(units=64)(X)  # 64\nH1 = ReLU()(X_W1)\n\n# 2nd hidden layer\nH1_W2 = Dense(units=64)(H1)  # 64\nH2 = ReLU()(H1_W2)\n\n# output layer\nH2_W3 = Dense(units=output_shape)(H2)  # 4\nH3 = Softmax()(H2_W3)\n\nmodel_output = H3\n\n# create model\nmodel = Model(inputs=[model_input], outputs=[model_output])\n\n# loss function & optimizer\nmodel.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\n# show model construction\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.4 Train the model","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import CSVLogger\n\ncsv_logger = CSVLogger('./training_log.csv')\n\n# training setting\nepochs = 25\nbatch_size = 32\n\n# training!\nhistory = model.fit(X_train, y_train, \n                    epochs=epochs, \n                    batch_size=batch_size, \n                    callbacks=[csv_logger],\n                    validation_data = (X_test, y_test))\nprint('training finish')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.5 Predict on testset","metadata":{}},{"cell_type":"code","source":"## predict\npred_result = model.predict(X_test, batch_size=128)\npred_result[:5]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_result = label_decode(label_encoder, pred_result)\npred_result[:5]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's take a look at the training log\ntraining_log = pd.DataFrame()\ntraining_log = pd.read_csv(\"./training_log.csv\")\ntraining_log","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. High-dimension Visualization: t-SNE","metadata":{}},{"cell_type":"markdown","source":"### 6.1 Prepare visualizing target","metadata":{}},{"cell_type":"code","source":"word_list = ['happy', 'angry', 'data', 'mining']\n\ntopn = 5\nhappy_words = ['happy'] + [word_ for word_, sim_ in w2v_google_model.most_similar('happy', topn=topn)]\nangry_words = ['angry'] + [word_ for word_, sim_ in w2v_google_model.most_similar('angry', topn=topn)]        \ndata_words = ['data'] + [word_ for word_, sim_ in w2v_google_model.most_similar('data', topn=topn)]        \nmining_words = ['mining'] + [word_ for word_, sim_ in w2v_google_model.most_similar('mining', topn=topn)]        \n\nprint('happy_words: ', happy_words)\nprint('angry_words: ', angry_words)\nprint('data_words: ', data_words)\nprint('mining_words: ', mining_words)\n\ntarget_words = happy_words + angry_words + data_words + mining_words\nprint('\\ntarget words: ')\nprint(target_words)\n\nprint('\\ncolor list:')\ncn = topn + 1\ncolor = ['b'] * cn + ['g'] * cn + ['r'] * cn + ['y'] * cn\nprint(color)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.2 Plot using t-SNE (2-dimension)","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom sklearn.manifold import TSNE\n\n## w2v model\nmodel = w2v_google_model\n\n## prepare training word vectors\nsize = 200\ntarget_size = len(target_words)\nall_word = list(model.index_to_key)\nword_train = target_words + all_word[:size]\nX_train = model[word_train]\n\n## t-SNE model\ntsne = TSNE(n_components=2, metric='cosine', random_state=28)\n\n## training\nX_tsne = tsne.fit_transform(X_train)\n\n## plot the result\nplt.figure(figsize=(7.5, 7.5), dpi=115)\nplt.scatter(X_tsne[:target_size, 0], X_tsne[:target_size, 1], c=color)\nfor label, x, y in zip(target_words, X_tsne[:target_size, 0], X_tsne[:target_size, 1]):\n    plt.annotate(label, xy=(x,y), xytext=(0,0),  textcoords='offset points')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"pd.read_csv('../input/dm2022-isa5810-lab2-homework/sampleSubmission.csv').shape","metadata":{"execution":{"iopub.status.busy":"2022-11-21T14:21:34.775314Z","iopub.execute_input":"2022-11-21T14:21:34.775829Z","iopub.status.idle":"2022-11-21T14:21:35.263991Z","shell.execute_reply.started":"2022-11-21T14:21:34.775768Z","shell.execute_reply":"2022-11-21T14:21:35.262575Z"},"trusted":true},"execution_count":null,"outputs":[]}]}