{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":35308,"databundleVersionId":3712025,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom string import punctuation\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom tqdm import tqdm\nimport re\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\nimport datetime\nfrom scipy import sparse\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom transformers import BertTokenizer\n\n%matplotlib inline\nimport datasets, transformers\n\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\nfrom transformers import AutoModelForMaskedLM\nos.environ[\"WANDB_DISABLED\"] = \"true\"\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom transformers import TFBertModel\nimport transformers\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleanup_text(text):\n    words = re.sub(pattern = '[^a-zA-Z]',repl = ' ', string = text)\n    words = words.lower()\n    return words\n\ndef EffectDatasetMapFunction(input_ids, attn_masks, labels):\n    return {\n        'input_ids': input_ids,\n        'attention_mask': attn_masks\n    }, labels\n\n# Texts, tokenizer inputs and Maxlength of inputs\ndef bert_encode(texts, tokenizer, max_len=MAX_LEN):\n    input_ids = []\n    token_type_ids = []\n    attention_mask = []\n    \n    for text in texts:\n        token = tokenizer(text, max_length=max_len, truncation=True, padding='max_length',\n                         add_special_tokens=True)\n        input_ids.append(token['input_ids'])\n        token_type_ids.append(token['token_type_ids'])\n        attention_mask.append(token['attention_mask'])\n    \n    return np.array(input_ids), np.array(token_type_ids), np.array(attention_mask)\n\ndef scheduler(epoch):\n    learning_rate = 1e-5\n    if epoch == 0:\n        return learning_rate\n    else:\n        return learning_rate * (0.8**epoch)\n    \ndef generate_training_data(df, ids, masks, tokenizer):\n    for i, text in tqdm(enumerate(df['discourse_text'])):\n        tokenized_text = tokenizer.encode_plus(\n            text,\n            max_length=256,\n            truncation=True,\n            padding='max_length',\n            add_special_tokens=True,\n            return_tensors='tf'\n        )\n        ids[i, :] = tokenized_text.input_ids\n        masks[i, :] = tokenized_text.attention_mask\n    return ids, masks\n\ndef create_data(train, tokenizer):\n    X_input_ids = np.zeros((len(train), 256))\n    X_attn_masks = np.zeros((len(train), 256))\n\n    X_input_ids, X_attn_masks = generate_training_data(train, X_input_ids, X_attn_masks, tokenizer)\n    labels = np.zeros((len(train), 3))\n    labels[np.arange(len(train)), train['discourse_effectiveness'].values] = 1 # one-hot encoded target tensor\n    \n    # creating a data pipeline using tensorflow dataset utility, creates batches of data for easy loading...\n    dataset = tf.data.Dataset.from_tensor_slices((X_input_ids, X_attn_masks, labels))\n    dataset = dataset.map(EffectDatasetMapFunction) # converting to required format for tensorflow dataset\n    dataset = dataset.shuffle(10000).batch(16, drop_remainder=True) # batch size, drop any left out tensor\n\n    p = 0.8\n    train_size = int((len(train)//16)*p) # for each 16 batch of data we will have len(train)//16 samples, take 80% of that for train.\n\n    train_dataset = dataset.take(train_size)\n    val_dataset = dataset.skip(train_size)\n    \n    return train_dataset, val_dataset","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntext_preprocessed = train['discourse_text'].apply(cleanup_text)\ntext_preprocessed\ntrain['text_preprocessed'] = text_preprocessed\ndisplay(train.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.BertTokenizer.from_pretrained('../input/huggingface-bert/bert-base-cased')\n# Save the loaded tokenizer locally\ntokenizer.save_pretrained('.')\nsep = tokenizer.sep_token\ntrain['inputs'] = train.discourse_type + sep +train.discourse_text\nbin_map = {\"discourse_effectiveness\": {\"Ineffective\": 0, \"Adequate\": 1, \"Effective\": 2}}\ntrain = train.replace(bin_map)\ntrain.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = train.tail(7002)\ntrain = train.iloc[:-7002]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test.drop(\"discourse_effectiveness\", axis=1)\ny_test = test[\"discourse_effectiveness\"]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_tokenizer = BertTokenizer.from_pretrained('bert-base-cased')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset, valid_dataset = create_data(train, bert_tokenizer)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = TFBertModel.from_pretrained('bert-base-cased') # bert base model with pretrained weights","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining 2 input layers for input_ids and attn_masks\ninput_ids = tf.keras.layers.Input(shape=(256,), name='input_ids', dtype='int32')\nattn_masks = tf.keras.layers.Input(shape=(256,), name='attention_mask', dtype='int32')\n\nbert_embds = model.bert(input_ids, attention_mask=attn_masks)[1] # 0 -> activation layer (3D), 1 -> pooled output layer (2D)\nintermediate_layer = tf.keras.layers.Dense(512, activation='relu', name='intermediate_layer')(bert_embds)\noutput_layer = tf.keras.layers.Dense(3, activation='softmax', name='output_layer')(intermediate_layer) # softmax -> calcs probs of classes\n\nsentiment_model = tf.keras.Model(inputs=[input_ids, attn_masks], outputs=output_layer)\nsentiment_model.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optim = tf.keras.optimizers.Adam(learning_rate=0.001,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=1e-07,\n    amsgrad=False,\n    name='adam')\nmetric = [tf.keras.metrics.CategoricalAccuracy('accuracy'), tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall')]\nsentiment_model.compile(optimizer=optim, loss = tf.keras.losses.categorical_crossentropy, metrics=metric)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)\ntrain_history = sentiment_model.fit(\n    train_dataset,\n    validation_data=valid_dataset,\n    epochs=30,\n    callbacks = [EarlyStopping(monitor='val_loss', patience=8, mode='min', restore_best_weights=True),\n                     ModelCheckpoint('bert_uspppm.h5', monitor='val_loss', mode='min', save_best_only=True, save_weights_only=True),\n                     callback_lr],\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_history.history['accuracy'])\nplt.plot(train_history.history['val_accuracy'])\nplt.title('Accuracy Curves with BERT')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_history.history['loss'])\nplt.plot(train_history.history['val_loss'])\nplt.title('Loss Curves with BERT')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_input_ids = np.zeros((len(X_test), 256)) \nX_attn_masks = np.zeros((len(X_test), 256))\n\nX_input_ids, X_attn_masks = generate_training_data(X_test, X_input_ids, X_attn_masks, bert_tokenizer)\n\npreds = sentiment_model.predict([X_input_ids, X_attn_masks])\npreds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(preds, axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\n\nconf_matrix = confusion_matrix(y_test, y_pred)\nprint(\"Confusion Matrix:\")\nprint(conf_matrix)","metadata":{},"execution_count":null,"outputs":[]}]}