{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:03:09.492540Z","iopub.execute_input":"2022-01-01T19:03:09.493050Z","iopub.status.idle":"2022-01-01T19:03:09.521692Z","shell.execute_reply.started":"2022-01-01T19:03:09.492959Z","shell.execute_reply":"2022-01-01T19:03:09.520971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove Emoji","metadata":{}},{"cell_type":"code","source":"from string import punctuation\n\nimport re\n\ndef remove_emojis(data):\n    emoj = re.compile(\"[\"\n        u\"\\U00002700-\\U000027BF\"  # Dingbats\n        u\"\\U0001F600-\\U0001F64F\"  # Emoticons\n        u\"\\U00002600-\\U000026FF\"  # Miscellaneous Symbols\n        u\"\\U0001F300-\\U0001F5FF\"  # Miscellaneous Symbols And Pictographs\n        u\"\\U0001F900-\\U0001F9FF\"  # Supplemental Symbols and Pictographs\n        u\"\\U0001FA70-\\U0001FAFF\"  # Symbols and Pictographs Extended-A\n        u\"\\U0001F680-\\U0001F6FF\"  # Transport and Map Symbols\n                      \"]+\", re.UNICODE)\n    return re.sub(emoj, '', data)\n\n\ndef remove_punctuation_emoji(value):\n    value = remove_emojis(value)\n    return ''.join([c for c in value if (c not in punctuation)])","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:03:09.523362Z","iopub.execute_input":"2022-01-01T19:03:09.523883Z","iopub.status.idle":"2022-01-01T19:03:09.531482Z","shell.execute_reply.started":"2022-01-01T19:03:09.523842Z","shell.execute_reply":"2022-01-01T19:03:09.530461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Data Structure","metadata":{}},{"cell_type":"code","source":"data = {\n    \"train\":{\n        \"<id>\":{\n            \"emotion\":\"\",\n            \"emotion_class\":1,\n            \"text\":\"\",\n            \"text_calss\":[],\n        }  \n    },\n    \"test\":{\n        \"<id>\":{\n            \"text\":\"\",\n            \"text_calss\":[],\n        },\n    },\n    \"emotion_map\":{},\n    \"tweet_map\":{},\n    \"max_voc\":1,\n    \"max_l\":1,\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:03:09.575284Z","iopub.execute_input":"2022-01-01T19:03:09.575594Z","iopub.status.idle":"2022-01-01T19:03:09.580819Z","shell.execute_reply.started":"2022-01-01T19:03:09.575564Z","shell.execute_reply":"2022-01-01T19:03:09.579873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Json","metadata":{}},{"cell_type":"code","source":"import json\nwith open(\"../input/dm2021-lab2-hw2/tweets_DM.json\",\"r\") as file:\n    tweets_json_list = [eval(f) for f in file.readlines()]","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:03:09.587987Z","iopub.execute_input":"2022-01-01T19:03:09.588467Z","iopub.status.idle":"2022-01-01T19:04:21.860468Z","shell.execute_reply.started":"2022-01-01T19:03:09.588437Z","shell.execute_reply":"2022-01-01T19:04:21.859538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweets_user = {u['_source'][\"tweet\"][\"tweet_id\"]:remove_punctuation_emoji(u['_source'][\"tweet\"][\"text\"]) for u in tweets_json_list}","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:04:21.862249Z","iopub.execute_input":"2022-01-01T19:04:21.862498Z","iopub.status.idle":"2022-01-01T19:04:57.505575Z","shell.execute_reply.started":"2022-01-01T19:04:21.862462Z","shell.execute_reply":"2022-01-01T19:04:57.504803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Emotion","metadata":{}},{"cell_type":"code","source":"import csv\n\nemotion = None\nwith open(\"../input/dm2021-lab2-hw2/emotion.csv\",newline='') as csvfile:\n    reader = csv.reader(csvfile, delimiter=',')\n    _ = next(reader)\n    emotion = {rows[0]:rows[1] for rows in reader} #row[0]:key row[1]:data, dict as a key value","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:04:57.506980Z","iopub.execute_input":"2022-01-01T19:04:57.507226Z","iopub.status.idle":"2022-01-01T19:04:59.061816Z","shell.execute_reply.started":"2022-01-01T19:04:57.507191Z","shell.execute_reply":"2022-01-01T19:04:59.061083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Data Identification","metadata":{}},{"cell_type":"code","source":"data_identification = None\n\nwith open(\"../input/dm2021-lab2-hw2/data_identification.csv\",newline='') as csvfile:\n    reader = csv.reader(csvfile, delimiter=',')\n    _ = next(reader) # iterate once to skip the first line\n    data_identification = {rows[0]: rows[1] for rows in reader} ","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:04:59.063828Z","iopub.execute_input":"2022-01-01T19:04:59.064087Z","iopub.status.idle":"2022-01-01T19:05:00.948831Z","shell.execute_reply.started":"2022-01-01T19:04:59.064052Z","shell.execute_reply":"2022-01-01T19:05:00.948102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Data","metadata":{}},{"cell_type":"code","source":"for k in data_identification.keys():\n    if data_identification[k] == \"train\":\n        data[\"train\"][k] = {}\n    elif data_identification[k] == \"test\":\n        data[\"test\"][k] = {}","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:00.950103Z","iopub.execute_input":"2022-01-01T19:05:00.950353Z","iopub.status.idle":"2022-01-01T19:05:02.517147Z","shell.execute_reply.started":"2022-01-01T19:05:00.950321Z","shell.execute_reply":"2022-01-01T19:05:02.516368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_to_remove = []\nfor k in data[\"train\"].keys():\n    if k in tweets_user:\n        data[\"train\"][k][\"text\"] = tweets_user[k]\n    else:\n        key_to_remove.append(k)\n\nfor k in key_to_remove:\n    data[\"train\"].pop(k)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:02.518464Z","iopub.execute_input":"2022-01-01T19:05:02.518730Z","iopub.status.idle":"2022-01-01T19:05:04.350613Z","shell.execute_reply.started":"2022-01-01T19:05:02.518697Z","shell.execute_reply":"2022-01-01T19:05:04.349869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_to_remove = []\nfor k in data[\"test\"].keys():\n    if k in tweets_user:\n        data[\"test\"][k][\"text\"] = tweets_user[k]\n    else:\n        key_to_remove.append(k)\n\nfor k in key_to_remove:\n    data[\"test\"].pop(k)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:04.351964Z","iopub.execute_input":"2022-01-01T19:05:04.352226Z","iopub.status.idle":"2022-01-01T19:05:04.905069Z","shell.execute_reply.started":"2022-01-01T19:05:04.352193Z","shell.execute_reply":"2022-01-01T19:05:04.904116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_to_remove = []\nfor k in data[\"train\"].keys():\n    if k in emotion:\n        data[\"train\"][k][\"emotion\"] = emotion[k]\n    else:\n        key_to_remove.append(k)\n    \nfor k in key_to_remove:\n    data[\"train\"].pop(k)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:04.906254Z","iopub.execute_input":"2022-01-01T19:05:04.906516Z","iopub.status.idle":"2022-01-01T19:05:06.567750Z","shell.execute_reply.started":"2022-01-01T19:05:04.906482Z","shell.execute_reply":"2022-01-01T19:05:06.567043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encode Emotion","metadata":{}},{"cell_type":"code","source":"class_emotion = []\n\nfor m in emotion.keys():\n    if emotion[m] not in class_emotion:\n        class_emotion.append(emotion[m])","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:06.569072Z","iopub.execute_input":"2022-01-01T19:05:06.569314Z","iopub.status.idle":"2022-01-01T19:05:07.067528Z","shell.execute_reply.started":"2022-01-01T19:05:06.569282Z","shell.execute_reply":"2022-01-01T19:05:07.066823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emotion_map = {v:i for i,v in enumerate(class_emotion)}","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:07.070459Z","iopub.execute_input":"2022-01-01T19:05:07.070731Z","iopub.status.idle":"2022-01-01T19:05:07.074801Z","shell.execute_reply.started":"2022-01-01T19:05:07.070697Z","shell.execute_reply":"2022-01-01T19:05:07.073974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in data[\"train\"].keys():\n    data[\"train\"][k][\"emotion_class\"] = emotion_map[data[\"train\"][k][\"emotion\"]]","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:07.076051Z","iopub.execute_input":"2022-01-01T19:05:07.076896Z","iopub.status.idle":"2022-01-01T19:05:08.236534Z","shell.execute_reply.started":"2022-01-01T19:05:07.076856Z","shell.execute_reply":"2022-01-01T19:05:08.235831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"emotion_map\"] = emotion_map","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:08.237650Z","iopub.execute_input":"2022-01-01T19:05:08.237903Z","iopub.status.idle":"2022-01-01T19:05:08.244048Z","shell.execute_reply.started":"2022-01-01T19:05:08.237870Z","shell.execute_reply":"2022-01-01T19:05:08.242829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze Text","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm ","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:08.245184Z","iopub.execute_input":"2022-01-01T19:05:08.245563Z","iopub.status.idle":"2022-01-01T19:05:08.252740Z","shell.execute_reply.started":"2022-01-01T19:05:08.245525Z","shell.execute_reply":"2022-01-01T19:05:08.252047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweets_data = {}\nfor k in tqdm(tweets_user.keys()):\n    users_text_list = tweets_user[k].split(\" \")\n    for t in users_text_list:\n        t = t.lower()\n        if t not in tweets_data:\n            tweets_data[t] = 1\n        else:\n            tweets_data[t] = tweets_data[t] + 1","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:08.253809Z","iopub.execute_input":"2022-01-01T19:05:08.254307Z","iopub.status.idle":"2022-01-01T19:05:27.132239Z","shell.execute_reply.started":"2022-01-01T19:05:08.254266Z","shell.execute_reply":"2022-01-01T19:05:27.131363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_key = []\nfor k in tweets_data.keys():\n    if '\\r' in k:\n        remove_key.append(k)\n    elif r'\\u' in k:\n        remove_key.append(k)\n    elif '\"\\\"' in k:\n        remove_key.append(k)\n    elif k.isdigit():\n        remove_key.append(k)\n    elif k == '':\n        remove_key.append(k)\n    elif k[0].isdigit():\n        remove_key.append(k)\n    elif len(k)>10:\n        remove_key.append(k)\n    elif '0' in k:\n        remove_key.append(k)\n    elif '1' in k:\n        remove_key.append(k)\n    elif '2' in k:\n        remove_key.append(k)\n    elif '3' in k:\n        remove_key.append(k)\n    elif '4' in k:\n        remove_key.append(k)\n    elif '5' in k:\n        remove_key.append(k)\n    elif '6' in k:\n        remove_key.append(k)\n    elif '7' in k:\n        remove_key.append(k)\n    elif '8' in k:\n        remove_key.append(k)\n    elif '9' in k:\n        remove_key.append(k)     \n        \nfor i in remove_key:\n    tweets_data.pop(i)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:27.133645Z","iopub.execute_input":"2022-01-01T19:05:27.134381Z","iopub.status.idle":"2022-01-01T19:05:28.607717Z","shell.execute_reply.started":"2022-01-01T19:05:27.134343Z","shell.execute_reply":"2022-01-01T19:05:28.606994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_tweets_key = sorted(tweets_data.keys())","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:28.609738Z","iopub.execute_input":"2022-01-01T19:05:28.610173Z","iopub.status.idle":"2022-01-01T19:05:29.108227Z","shell.execute_reply.started":"2022-01-01T19:05:28.610127Z","shell.execute_reply":"2022-01-01T19:05:29.107343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweet_map  = {'':0}\nfor i,v in enumerate(sorted_tweets_key):\n    tweet_map[v] = i + 1","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:29.109770Z","iopub.execute_input":"2022-01-01T19:05:29.110039Z","iopub.status.idle":"2022-01-01T19:05:29.406596Z","shell.execute_reply.started":"2022-01-01T19:05:29.110005Z","shell.execute_reply":"2022-01-01T19:05:29.405900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_max = 0\nfor k in tqdm(tweets_user.keys()):\n    users_text_list = tweets_user[k].split(\" \")\n    embedded_text = []\n    for t in users_text_list:\n        t = t.lower()\n        if t in tweet_map:\n            embedded_text.append(tweet_map[t])\n    if len(embedded_text)>l_max:\n        l_max = len(embedded_text)\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:29.407782Z","iopub.execute_input":"2022-01-01T19:05:29.408064Z","iopub.status.idle":"2022-01-01T19:05:49.329853Z","shell.execute_reply.started":"2022-01-01T19:05:29.408030Z","shell.execute_reply":"2022-01-01T19:05:49.329170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in tqdm(data[\"train\"].keys()):\n    users_text_list = data[\"train\"][k][\"text\"].split(\" \")\n    embedded_text = []\n    for t in users_text_list:\n        t = t.lower()\n        if t in tweet_map:\n            embedded_text.append(tweet_map[t])\n    while len(embedded_text)<l_max:\n        embedded_text.append(0)\n    data[\"train\"][k][\"text_class\"] = embedded_text","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:05:49.331075Z","iopub.execute_input":"2022-01-01T19:05:49.331542Z","iopub.status.idle":"2022-01-01T19:06:49.785434Z","shell.execute_reply.started":"2022-01-01T19:05:49.331503Z","shell.execute_reply":"2022-01-01T19:06:49.784734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in tqdm(data[\"test\"].keys()):\n    users_text_list = data[\"test\"][k][\"text\"].split(\" \")\n    embedded_text = []\n    for t in users_text_list:\n        t = t.lower()\n        if t in tweet_map:\n            embedded_text.append(tweet_map[t])\n    while len(embedded_text)<l_max:\n        embedded_text.append(0)\n    data[\"test\"][k][\"text_class\"] = embedded_text\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:06:49.786585Z","iopub.execute_input":"2022-01-01T19:06:49.787374Z","iopub.status.idle":"2022-01-01T19:07:05.946559Z","shell.execute_reply.started":"2022-01-01T19:06:49.787334Z","shell.execute_reply":"2022-01-01T19:07:05.945866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"tweet_map\"] = tweet_map\ndata[\"max_voc\"] = len(tweet_map)\ndata[\"max_l\"] = l_max","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:05.947841Z","iopub.execute_input":"2022-01-01T19:07:05.948547Z","iopub.status.idle":"2022-01-01T19:07:05.953317Z","shell.execute_reply.started":"2022-01-01T19:07:05.948505Z","shell.execute_reply":"2022-01-01T19:07:05.952385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Trained Data","metadata":{}},{"cell_type":"code","source":"import torch \n\ntorch.load(\"../input/traineddmhw2/model_full.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:05.954766Z","iopub.execute_input":"2022-01-01T19:07:05.955069Z","iopub.status.idle":"2022-01-01T19:07:14.142448Z","shell.execute_reply.started":"2022-01-01T19:07:05.955029Z","shell.execute_reply":"2022-01-01T19:07:14.141737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data import SubsetRandomSampler\nfrom torch import nn","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.143811Z","iopub.execute_input":"2022-01-01T19:07:14.144264Z","iopub.status.idle":"2022-01-01T19:07:14.148784Z","shell.execute_reply.started":"2022-01-01T19:07:14.144228Z","shell.execute_reply":"2022-01-01T19:07:14.148059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TWEETDataset_test(Dataset):\n    def __init__(self, name, feature):\n        self.name = name\n        self.feature = [torch.tensor(i) for i in feature]\n\n    def __len__(self):\n        return len(self.feature)\n\n    def __getitem__(self, idx):\n        return self.name[idx], self.feature[idx]","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.150091Z","iopub.execute_input":"2022-01-01T19:07:14.150951Z","iopub.status.idle":"2022-01-01T19:07:14.159312Z","shell.execute_reply.started":"2022-01-01T19:07:14.150911Z","shell.execute_reply":"2022-01-01T19:07:14.158647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This script was modified from\n# https://github.com/TalwalkarLab/leaf/blob/master/models/shakespeare/stacked_lstm.py\nimport torch\n\nclass LSTM_model(torch.nn.Module):\n    def __init__(self, n_vocab=80, output = 10, embedding_dim=64, hidden_dim_1=256, nb_layers_1=1):\n        super(LSTM_model, self).__init__()\n\n        self.embedding_dim = embedding_dim\n        self.hidden_dim_1 = hidden_dim_1\n\n        self.embeddings = torch.nn.Embedding(n_vocab, embedding_dim)\n        self.lstm_1 = torch.nn.LSTM(embedding_dim, hidden_dim_1, nb_layers_1)\n        self.hidden2out = torch.nn.Linear(hidden_dim_1, output)\n\n    def forward(self, seq_in):\n        embeddings = self.embeddings(seq_in.t())\n        lstm_out, h_state1 = self.lstm_1(embeddings)\n        \n        ht = lstm_out[-1]\n        out = self.hidden2out(ht)\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.160707Z","iopub.execute_input":"2022-01-01T19:07:14.161076Z","iopub.status.idle":"2022-01-01T19:07:14.169804Z","shell.execute_reply.started":"2022-01-01T19:07:14.161039Z","shell.execute_reply":"2022-01-01T19:07:14.168880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(model, test_data):\n    model.to(device)\n    model.eval()\n    \n    names = []\n    predicts = []\n\n    with torch.no_grad():\n        for name, data in tqdm(test_data):\n\n            data = data.to(device)\n            pred = model(data)\n            _, predicted = torch.max(pred, -1)\n            \n            names.append(name)\n            predicts.append(predicted)\n\n    return names, predicts","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.171123Z","iopub.execute_input":"2022-01-01T19:07:14.171828Z","iopub.status.idle":"2022-01-01T19:07:14.180513Z","shell.execute_reply.started":"2022-01-01T19:07:14.171792Z","shell.execute_reply":"2022-01-01T19:07:14.179772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\")\nbatch_size = 4\nLR = 0.001\nepoch = 2","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.183545Z","iopub.execute_input":"2022-01-01T19:07:14.183862Z","iopub.status.idle":"2022-01-01T19:07:14.188317Z","shell.execute_reply.started":"2022-01-01T19:07:14.183771Z","shell.execute_reply":"2022-01-01T19:07:14.187547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x = []\ntrain_y = []\n\ntest_name = []\ntest_x = []","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.193053Z","iopub.execute_input":"2022-01-01T19:07:14.193299Z","iopub.status.idle":"2022-01-01T19:07:14.197025Z","shell.execute_reply.started":"2022-01-01T19:07:14.193273Z","shell.execute_reply":"2022-01-01T19:07:14.196192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in data[\"train\"].keys():\n    train_x.append(data[\"train\"][k]['text_class'])\n    train_y.append(data[\"train\"][k]['emotion_class'])\n        \nfor k in data[\"test\"].keys():\n    test_name.append(k)\n    test_x.append(data[\"test\"][k]['text_class'])","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:14.198592Z","iopub.execute_input":"2022-01-01T19:07:14.198936Z","iopub.status.idle":"2022-01-01T19:07:15.619636Z","shell.execute_reply.started":"2022-01-01T19:07:14.198900Z","shell.execute_reply":"2022-01-01T19:07:15.618918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total train data: {}\".format(len(train_x)))\nprint(\"Total test data: {}\".format(len(test_x)))","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:15.620874Z","iopub.execute_input":"2022-01-01T19:07:15.621155Z","iopub.status.idle":"2022-01-01T19:07:15.627841Z","shell.execute_reply.started":"2022-01-01T19:07:15.621121Z","shell.execute_reply":"2022-01-01T19:07:15.626805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TWEETDataset_test(name=test_name, feature=test_x)\ntest_dataloader = DataLoader(test_dataset, batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:15.629225Z","iopub.execute_input":"2022-01-01T19:07:15.629789Z","iopub.status.idle":"2022-01-01T19:07:22.856631Z","shell.execute_reply.started":"2022-01-01T19:07:15.629748Z","shell.execute_reply":"2022-01-01T19:07:22.855919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM_model(n_vocab=data[\"max_voc\"], output = 8, embedding_dim=128, hidden_dim_1=256, nb_layers_1=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:22.857919Z","iopub.execute_input":"2022-01-01T19:07:22.858180Z","iopub.status.idle":"2022-01-01T19:07:23.531340Z","shell.execute_reply.started":"2022-01-01T19:07:22.858147Z","shell.execute_reply":"2022-01-01T19:07:23.530582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name, predict = tqdm(test(model=model,test_data=test_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:07:23.532637Z","iopub.execute_input":"2022-01-01T19:07:23.532916Z","iopub.status.idle":"2022-01-01T19:10:52.947586Z","shell.execute_reply.started":"2022-01-01T19:07:23.532880Z","shell.execute_reply":"2022-01-01T19:10:52.946706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_emotion_by_class(value):\n    return list(data[\"emotion_map\"] .keys())[list(data[\"emotion_map\"].values()).index(value)]","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:10:52.949104Z","iopub.execute_input":"2022-01-01T19:10:52.949371Z","iopub.status.idle":"2022-01-01T19:10:52.954434Z","shell.execute_reply.started":"2022-01-01T19:10:52.949335Z","shell.execute_reply":"2022-01-01T19:10:52.953369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"result = {}\nfor n,p in zip(name, predict):\n    p = p.cpu().tolist()\n    for i,v in enumerate(n):\n        result[v] = get_emotion_by_class(p[i])","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:14:45.186931Z","iopub.execute_input":"2022-01-01T19:14:45.187564Z","iopub.status.idle":"2022-01-01T19:14:47.802949Z","shell.execute_reply.started":"2022-01-01T19:14:45.187523Z","shell.execute_reply":"2022-01-01T19:14:47.802145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"result_id = list(result.keys())\nresult_values = list(result.values())","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:19:29.203264Z","iopub.execute_input":"2022-01-01T19:19:29.203992Z","iopub.status.idle":"2022-01-01T19:19:29.222422Z","shell.execute_reply.started":"2022-01-01T19:19:29.203926Z","shell.execute_reply":"2022-01-01T19:19:29.221413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:19:33.338890Z","iopub.execute_input":"2022-01-01T19:19:33.341468Z","iopub.status.idle":"2022-01-01T19:19:33.346545Z","shell.execute_reply.started":"2022-01-01T19:19:33.341422Z","shell.execute_reply":"2022-01-01T19:19:33.345663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_data = pandas.DataFrame({\"id\":result_id, \"emotion\":result_values})\nsave_data.to_csv(\"./Submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:30.808435Z","iopub.execute_input":"2022-01-01T19:20:30.808771Z","iopub.status.idle":"2022-01-01T19:20:31.749438Z","shell.execute_reply.started":"2022-01-01T19:20:30.808728Z","shell.execute_reply":"2022-01-01T19:20:31.748598Z"},"trusted":true},"execution_count":null,"outputs":[]}]}