{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-20T16:24:24.685893Z","iopub.execute_input":"2022-11-20T16:24:24.686334Z","iopub.status.idle":"2022-11-20T16:24:24.714265Z","shell.execute_reply.started":"2022-11-20T16:24:24.686247Z","shell.execute_reply":"2022-11-20T16:24:24.713305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/huggingface/transformers","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:24:24.716072Z","iopub.execute_input":"2022-11-20T16:24:24.716687Z","iopub.status.idle":"2022-11-20T16:25:11.237659Z","shell.execute_reply.started":"2022-11-20T16:24:24.71663Z","shell.execute_reply":"2022-11-20T16:25:11.236496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom datasets import load_dataset\nfrom datasets import Dataset\n\ntweet_raw = load_dataset(\"json\", data_files=\"/kaggle/input/dm2022-isa5810-lab2-homework/tweets_DM.json\")\ntweet_raw = tweet_raw['train']['_source']","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:25:11.240317Z","iopub.execute_input":"2022-11-20T16:25:11.240733Z","iopub.status.idle":"2022-11-20T16:25:51.934121Z","shell.execute_reply.started":"2022-11-20T16:25:11.240689Z","shell.execute_reply":"2022-11-20T16:25:51.933048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tweet_df = pd.json_normalize(tweet_raw)\ntweet_df.columns= ['hashtags','tweet_id','text']\ntweet_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:25:51.940491Z","iopub.execute_input":"2022-11-20T16:25:51.940969Z","iopub.status.idle":"2022-11-20T16:26:06.056303Z","shell.execute_reply.started":"2022-11-20T16:25:51.94094Z","shell.execute_reply":"2022-11-20T16:26:06.055334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nOther data\n/kaggle/input/dm2022-isa5810-lab2-homework/sampleSubmission.csv\n/kaggle/input/dm2022-isa5810-lab2-homework/data_identification.csv\n/kaggle/input/dm2022-isa5810-lab2-homework/emotion.csv\n\"\"\"\ndata_iden = pd.read_csv('/kaggle/input/dm2022-isa5810-lab2-homework/data_identification.csv')\ndata_label = pd.read_csv('/kaggle/input/dm2022-isa5810-lab2-homework/emotion.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:06.059456Z","iopub.execute_input":"2022-11-20T16:26:06.05985Z","iopub.status.idle":"2022-11-20T16:26:08.137764Z","shell.execute_reply.started":"2022-11-20T16:26:06.059814Z","shell.execute_reply":"2022-11-20T16:26:08.13678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_id = data_iden[data_iden.identification==\"train\"]\ntest_id = data_iden[data_iden.identification==\"test\"]","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:08.139127Z","iopub.execute_input":"2022-11-20T16:26:08.139496Z","iopub.status.idle":"2022-11-20T16:26:08.439416Z","shell.execute_reply.started":"2022-11-20T16:26:08.139455Z","shell.execute_reply":"2022-11-20T16:26:08.438409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##create training dataset\ntext  = tweet_df[['tweet_id','text']].drop_duplicates()\ntrain_df = text[text.tweet_id.isin(train_id.tweet_id)]\ntrain_df = train_df.join(data_label.set_index('tweet_id'), on='tweet_id')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:08.44075Z","iopub.execute_input":"2022-11-20T16:26:08.441106Z","iopub.status.idle":"2022-11-20T16:26:14.691692Z","shell.execute_reply.started":"2022-11-20T16:26:08.441071Z","shell.execute_reply":"2022-11-20T16:26:14.690606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dropna(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:14.693304Z","iopub.execute_input":"2022-11-20T16:26:14.693709Z","iopub.status.idle":"2022-11-20T16:26:15.313243Z","shell.execute_reply.started":"2022-11-20T16:26:14.693672Z","shell.execute_reply":"2022-11-20T16:26:15.312114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##create testing dataset\ntest_df = text[text.tweet_id.isin(test_id.tweet_id)]\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:15.314905Z","iopub.execute_input":"2022-11-20T16:26:15.315276Z","iopub.status.idle":"2022-11-20T16:26:15.854811Z","shell.execute_reply.started":"2022-11-20T16:26:15.315238Z","shell.execute_reply":"2022-11-20T16:26:15.85371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder(sparse = False)\nencoder = encoder.fit(train_df[['emotion']])\nprint(encoder.categories_)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:15.856442Z","iopub.execute_input":"2022-11-20T16:26:15.856827Z","iopub.status.idle":"2022-11-20T16:26:16.252339Z","shell.execute_reply.started":"2022-11-20T16:26:15.856791Z","shell.execute_reply":"2022-11-20T16:26:16.251327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom datasets import load_dataset\nfrom datasets import Dataset\n#dataset = load_dataset(\"json\", data_files=\"/kaggle/input/dm2022-isa5810-lab2-homework/tweets_DM.json\")\ndataset = Dataset.from_pandas(train_df[['text','emotion']])","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:16.256085Z","iopub.execute_input":"2022-11-20T16:26:16.257037Z","iopub.status.idle":"2022-11-20T16:26:16.759673Z","shell.execute_reply.started":"2022-11-20T16:26:16.256998Z","shell.execute_reply":"2022-11-20T16:26:16.758683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Available models can be found here: https://huggingface.co/models\nMODEL_NAME = 'distilbert-base-uncased'\n\nfrom transformers import AutoTokenizer # For tokenization\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\nfrom transformers import DataCollatorWithPadding\n\n# declare a collator to do padding during traning.\ndata_collator = DataCollatorWithPadding(tokenizer=tokenizer)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:16.761378Z","iopub.execute_input":"2022-11-20T16:26:16.761785Z","iopub.status.idle":"2022-11-20T16:26:34.798457Z","shell.execute_reply.started":"2022-11-20T16:26:16.761748Z","shell.execute_reply":"2022-11-20T16:26:34.797478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\npattern = re.compile('@(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\nx = train_df.text.tolist()[1]\nprint(x)\ndef trim(x):\n    x = x.replace('<LH>','')\n    x = pattern.sub('', x)\n    return (x)\n\nprint(trim(x))","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:34.800099Z","iopub.execute_input":"2022-11-20T16:26:34.801037Z","iopub.status.idle":"2022-11-20T16:26:34.876137Z","shell.execute_reply.started":"2022-11-20T16:26:34.800997Z","shell.execute_reply":"2022-11-20T16:26:34.875073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef preprocess(dataslice):\n    \"\"\" Input: a batch of your dataset\n        Example: { 'text': [['sentence1'], ['setence2'], ...],\n                   'label': ['label1', 'label2', ...] }\n    \"\"\"\n    \n    embed = tokenizer(dataslice['text'])\n    label = encoder.transform([[i] for i in dataslice['emotion']])\n    output = {}\n    output.update(embed)\n    output.update({'label':[list(i) for i in label]})\n    \n\n    \"\"\" Output: a batch of processed dataset\n        Example: { 'input_ids': ...,\n                   'attention_masks': ...,\n                   'label': ... }\n    \"\"\"\n    return output\n\ndataslice = dict()\ndataslice.update({'text':[[trim(i)] for i in train_df.text.tolist()]})\ndataslice.update({'emotion':train_df.emotion.tolist()})\n\nprocessed_data = dataset.map(lambda dataslice: preprocess(dataslice), batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:26:34.87772Z","iopub.execute_input":"2022-11-20T16:26:34.878083Z","iopub.status.idle":"2022-11-20T16:28:43.626839Z","shell.execute_reply.started":"2022-11-20T16:26:34.878047Z","shell.execute_reply":"2022-11-20T16:28:43.625899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Change to TFAutoModelForSequenceClassification if you're using tensoflow\nfrom transformers import AutoModelForSequenceClassification\n\nLABEL_COUNT = len(encoder.categories_[0])\n#print(LABEL_COUNT)\n\nmodel = AutoModelForSequenceClassification.from_pretrained('distilbert-base-uncased',num_labels = LABEL_COUNT)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:43:00.468452Z","iopub.execute_input":"2022-11-20T16:43:00.468835Z","iopub.status.idle":"2022-11-20T16:43:03.364133Z","shell.execute_reply.started":"2022-11-20T16:43:00.4688Z","shell.execute_reply":"2022-11-20T16:43:03.363182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments, Trainer\n\nimport random\nprocessed_data_test = processed_data.shard(num_shards=100, index=0)\n\ntrain_val_dataset = processed_data_test.train_test_split(test_size = 0.3)\n\n# Set and tune training properties\nLEARNING_RATE = 0.0001\nBATCH_SIZE = 10\nEPOCH = 20\ntraining_args = TrainingArguments(\n    output_dir = 'model',\n    learning_rate = LEARNING_RATE,\n    per_device_train_batch_size = BATCH_SIZE,\n    per_device_eval_batch_size = BATCH_SIZE,\n    num_train_epochs = EPOCH,\n    logging_steps = 1000,\n    weight_decay=0.05,  \n)\n\n# Now give all information to a trainer.\ntrainer = Trainer(\n    model = model,\n    args = training_args,\n    train_dataset = train_val_dataset[\"train\"],\n    eval_dataset = train_val_dataset[\"test\"],\n    tokenizer = tokenizer,\n    data_collator = data_collator,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:42:50.286015Z","iopub.execute_input":"2022-11-20T16:42:50.286372Z","iopub.status.idle":"2022-11-20T16:42:50.340952Z","shell.execute_reply.started":"2022-11-20T16:42:50.28634Z","shell.execute_reply":"2022-11-20T16:42:50.339904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"WANDB_DISABLED\"] = \"true\"\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:32:46.77674Z","iopub.execute_input":"2022-11-20T16:32:46.777214Z","iopub.status.idle":"2022-11-20T16:41:45.566293Z","shell.execute_reply.started":"2022-11-20T16:32:46.777171Z","shell.execute_reply":"2022-11-20T16:41:45.563605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_pretrained(os.path.join('model', 'finetuned'))\n!rm -rf checkpoints","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:42:04.020026Z","iopub.execute_input":"2022-11-20T16:42:04.020395Z","iopub.status.idle":"2022-11-20T16:42:04.835251Z","shell.execute_reply.started":"2022-11-20T16:42:04.020362Z","shell.execute_reply":"2022-11-20T16:42:04.834081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSequenceClassification\n\nmymodel = AutoModelForSequenceClassification.from_pretrained(os.path.join('model', 'finetuned'))","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:42:12.780991Z","iopub.execute_input":"2022-11-20T16:42:12.781345Z","iopub.status.idle":"2022-11-20T16:42:13.86195Z","shell.execute_reply.started":"2022-11-20T16:42:12.781313Z","shell.execute_reply":"2022-11-20T16:42:13.86055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load test data\ndef preprocess(dataslice):\n    \"\"\" Input: a batch of your dataset\n        Example: { 'text': [['sentence1'], ['setence2'], ...],\n                   'label': ['label1', 'label2', ...] }\n    \"\"\"\n    \n    embed = tokenizer(dataslice['text'])\n    output = {}\n    output.update(embed)\n    \n\n    \"\"\" Output: a batch of processed dataset\n        Example: { 'input_ids': ...,\n                   'attention_masks': ...,\n                   'label': ... }\n    \"\"\"\n    return output\n\ntest_data = Dataset.from_pandas(test_df[['text']])\n\n# preprocess\ntest_processed_data = test_data.map(lambda dataslice: preprocess(dataslice), batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:32:37.887363Z","iopub.status.idle":"2022-11-20T16:32:37.887735Z","shell.execute_reply.started":"2022-11-20T16:32:37.88754Z","shell.execute_reply":"2022-11-20T16:32:37.887557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get predictions\n# Transform the sentences into embeddings\nfrom torch import nn\nimport tensorflow as tf\n\npredict_label = []\nfor i in range(1,500):\n    test_input = tokenizer(test_processed_data['text'], truncation=True, padding=True, return_tensors=\"pt\")\n    test_logits = mymodel(**test_input).logits\n    test_predicts = nn.functional.softmax(test_logits, dim = -1)\n    predict_label_onehot = [list(tf.one_hot(tf.argmax(i),8).numpy()) for i in test_predicts.detach().numpy()]\n    predict_label = predict_label+ encoder.inverse_transform(predict_label_onehot).tolist()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:32:37.889919Z","iopub.status.idle":"2022-11-20T16:32:37.890402Z","shell.execute_reply.started":"2022-11-20T16:32:37.890159Z","shell.execute_reply":"2022-11-20T16:32:37.890183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we still recommend you to print out some predictions to check if the outputs are resonable and if you need to adjust your model at the end of every step.\n\nfor idx, (sent, level) in enumerate(zip(test_data, predict_label)):\n    if idx >= 20: break\n    print(f'{level[0]}: {sent}') ","metadata":{"execution":{"iopub.status.busy":"2022-11-20T16:32:37.895527Z","iopub.status.idle":"2022-11-20T16:32:37.896081Z","shell.execute_reply.started":"2022-11-20T16:32:37.89577Z","shell.execute_reply":"2022-11-20T16:32:37.895887Z"},"trusted":true},"execution_count":null,"outputs":[]}]}