{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom tqdm.autonotebook import tqdm\nimport os\nimport matplotlib.pyplot as plt\nimport json\nimport nltk\nimport re\nimport numpy as np\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom collections import OrderedDict, Counter\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split \nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n# nltk.download('stopwords')\n# nltk.download('punkt')\n# nltk.download('wordnet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Concatenate(filename, files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n\n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(Concatenate)\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(Concatenate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.drop(columns = ['pub_title','dataset_title', 'dataset_label','Id'], inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['cleaned_label'].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['dataset_title'].value_counts().plot(kind='bar', color='red', figsize = (24,20), fontsize = 10)\nplt.xlabel('labels')\nplt.ylabel('Total label count')\nplt.title('label count of each label type')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleaning_text(text): \n    text = re.sub(\"[^a-zA-Z]\",\" \",text) \n    text = re.sub('[!@#$_]', '', text)\n    text = text.replace(\"co\",\"\")\n    text = text.replace(\"http\",\"\")\n    text = ' '.join(text.split()) \n    text = text.lower() \n    return text\n\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: cleaning_text(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: cleaning_text(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = train_df.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    token_words= word_tokenize(str(text))\n    return \" \".join(token_words)\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: tokenize(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: tokenize(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stopwords_clean(text):\n    stop_words = set(stopwords.words('english'))\n    no_stopword_text = [w for w in str(text).split() if not w in stop_words]\n    return \" \".join(no_stopword_text)\n\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: stopwords_clean(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: stopwords_clean(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import nltk\n# from nltk.stem import WordNetLemmatizer\n# lemma= WordNetLemmatizer() \n\n# def lemmatize_text(text):\n#     lemma_text = [lemma.lemmatize(word) for word in text]\n#     return \"\".join(lemma_text)\n\n# train_df['texts_cleaned_lemmatized'] = train_df['text_cleaned_nostop'].progress_apply(lambda x: lemmatize_text(x))\n# sample_sub['texts_cleaned_lemmatized'] = sample_sub['text_cleaned_nostop'].progress_apply(lambda x: lemmatize_text(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = (train_df['text'].str.split().progress_apply(lambda x: OrderedDict.fromkeys(x).keys()).str.join(' '))\nsample_sub['text'] = (sample_sub['text'].str.split().progress_apply(lambda x: OrderedDict.fromkeys(x).keys()).str.join(' '))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df = train_df.copy() #ML\n# df = train_df.copy()       #BERT","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models.doc2vec import Doc2Vec, TaggedDocument\ntext_docs = [TaggedDocument(doc.split(' '), [i]) \n             for i, doc in enumerate(Train_df.text)]\nmodel = Doc2Vec(vector_size=128, min_count=3, epochs = 30)\n#instantiate model\nmodel = Doc2Vec(vector_size=128, window=2, min_count=3, workers=8, epochs = 50)\n#build vocab\nmodel.build_vocab(text_docs)\n#train model\nmodel.train(text_docs, total_examples=model.corpus_count, epochs=model.epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text2vec = [model.infer_vector((Train_df['text'][i].split(' '))) \n            for i in range(0,len(Train_df['text']))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_text2vec = [model.infer_vector((sample_sub['text'][i].split(' '))) \n            for i in range(0,len(sample_sub['text']))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtv= np.array(text2vec).tolist()\nTrain_df['text2vec'] = dtv\nTrain_df.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ttv= np.array(test_text2vec).tolist()\nsample_sub['text2vec'] = ttv\nsample_sub.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_sub.drop(columns = ['text','cleaned_text', 'cleaned_text_tokenized'], inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparce_matrix = text2vec\ny = Train_df.iloc[:,2].values \nX = sparce_matrix","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ny = encoder.fit_transform(y)\ny[:10]\n# X[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded = encoder.inverse_transform(y)\ndecoded[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=45)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Note - Accuracy is a bad metric of evaluation for the models as the classes are heavily imbalanced*","metadata":{}},{"cell_type":"markdown","source":"**Logistic Regression Classifier**","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\nlogistic = LogisticRegression(penalty = 'l1', solver='liblinear', C= 0.1, random_state = 45)\npipeline = Pipeline(steps=[('sc', sc),('logistic', logistic)])\npipeline.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{}},{"cell_type":"code","source":"y_pred = pipeline.predict(X_test)\nprint('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicts = []\nfor i in range (4):\n    lol  = np.array(test_text2vec[i]).reshape(1, -1)\n    pred_test =  pipeline.predict(lol)\n    X = encoder.inverse_transform(pred_test)\n    X = X.tolist()[0]\n    predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in Train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in Train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in Train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = Train_df[Train_df['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf.PredictionString = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf['PredictionString'] = sample_subf['PredictionString'].progress_apply(lambda x: cleaning_text(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**RandomForest Classifier**","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrfc1=RandomForestClassifier(n_estimators= 50, max_depth=15, bootstrap=True, random_state=45)\nrfc1.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{}},{"cell_type":"code","source":"y_pred = rfc1.predict(X_test)\nprint('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicts = []\nfor i in range (4):\n    lol  = np.array(test_text2vec[i]).reshape(1, -1)\n    pred_test =  rfc1.predict(lol)\n    X = encoder.inverse_transform(pred_test)\n    X = X.tolist()[0]\n    predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Support Vector Machine Classifier**","metadata":{}},{"cell_type":"code","source":"from sklearn.svm import SVC\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\n\nsc = StandardScaler()\nsvm = SVC(C=1,gamma='scale', kernel='linear')\npipe = Pipeline(steps=[('sc', sc),\n                       ('SVM', svm)])\n\npipe.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{}},{"cell_type":"code","source":"y_pred = pipe.predict(X_test)\nprint('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicts = []\nfor i in range (4):\n    lol  = np.array(test_text2vec[i]).reshape(1, -1)\n    pred_test =  pipe.predict(lol)\n    X = encoder.inverse_transform(pred_test)\n    X = X.tolist()[0]\n    predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Naive Bayes Classifier**","metadata":{}},{"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\ngnb = GaussianNB()\ngnb.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{}},{"cell_type":"code","source":"y_pred = gnb.predict(X_test)\nfrom sklearn import metrics\nprint(\"Accuracy:\",metrics.accuracy_score(y_test, y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicts = []\nfor i in range (4):\n    lol  = np.array(test_text2vec[i]).reshape(1, -1)\n    pred_test =  gnb.predict(lol)\n    X = encoder.inverse_transform(pred_test)\n    X = X.tolist()[0]\n    predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**BERT**","metadata":{}},{"cell_type":"markdown","source":"Encoding the Labels","metadata":{}},{"cell_type":"code","source":"# df.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# possible_labels = df.cleaned_label.unique()\n\n# label_dict = {}\n# for index, possible_label in enumerate(possible_labels):\n#     label_dict[possible_label] = index\n# label_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['label'] = df.cleaned_label.replace(label_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train and Validation Split","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n\n# X_train, X_val, y_train, y_val = train_test_split(df.index.values, \n#                                                   df.label.values, \n#                                                   test_size=0.15, \n#                                                   random_state=42)\n\n# df['data_type'] = ['not_set']*df.shape[0]\n\n# df.loc[X_train, 'data_type'] = 'train'\n# df.loc[X_val, 'data_type'] = 'val'\n\n# df.groupby(['cleaned_label', 'label', 'data_type']).count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BertTokenizer and Encoding the Data","metadata":{}},{"cell_type":"code","source":"# from tokenizers import Tokenizer\n# from tokenizers.models import BPE\n# from tokenizers.trainers import BpeTrainer\n# from tokenizers.pre_tokenizers import Whitespace","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install transformers\n# !pip install torchvision \n# import transformers\n# import torch\n# import torchvision\n# from torch.utils.data import TensorDataset \n# from transformers import BertForSequenceClassification","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', \n#                                           do_lower_case=True)\n                                          \n# encoded_data_train = tokenizer.batch_encode_plus(\n#     df[df.data_type=='train'].cleaned_text.values, \n#     add_special_tokens=True, \n#     return_attention_mask=True, \n#     pad_to_max_length=True, \n#     max_length=256, \n#     return_tensors='pt'\n# )\n\n# encoded_data_val = tokenizer.batch_encode_plus(\n#     df[df.data_type=='val'].cleaned_text.values, \n#     add_special_tokens=True, \n#     return_attention_mask=True, \n#     pad_to_max_length=True, \n#     max_length=256, \n#     return_tensors='pt'\n# )\n\n\n# input_ids_train = encoded_data_train['input_ids']\n# attention_masks_train = encoded_data_train['attention_mask']\n# labels_train = torch.tensor(df[df.data_type=='train'].label.values)\n\n# input_ids_val = encoded_data_val['input_ids']\n# attention_masks_val = encoded_data_val['attention_mask']\n# labels_val = torch.tensor(df[df.data_type=='val'].label.values)\n\n# dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)\n# dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(dataset_train), len(dataset_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BERT Pre-trained Model","metadata":{}},{"cell_type":"code","source":"# model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n#                                                       num_labels=len(label_dict),\n#                                                       output_attentions=False,\n#                                                       output_hidden_states=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n\n# batch_size = 3\n\n# dataloader_train = DataLoader(dataset_train, \n#                               sampler=RandomSampler(dataset_train), \n#                               batch_size=batch_size)\n\n# dataloader_validation = DataLoader(dataset_val, \n#                                    sampler=SequentialSampler(dataset_val), \n#                                    batch_size=batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Optimizer & Scheduler","metadata":{}},{"cell_type":"code","source":"# from transformers import AdamW, get_linear_schedule_with_warmup\n\n# optimizer = AdamW(model.parameters(),\n#                   lr=1e-5, \n#                   eps=1e-8)\n                  \n# epochs = 5\n\n# scheduler = get_linear_schedule_with_warmup(optimizer, \n#                                             num_warmup_steps=0,\n#                                             num_training_steps=len(dataloader_train)*epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Performance Metrics","metadata":{}},{"cell_type":"code","source":"# from sklearn.metrics import f1_score\n\n# def f1_score_func(preds, labels):\n#     preds_flat = np.argmax(preds, axis=1).flatten()\n#     labels_flat = labels.flatten()\n#     return f1_score(labels_flat, preds_flat, average='weighted')\n\n# def accuracy_per_class(preds, labels):\n#     label_dict_inverse = {v: k for k, v in label_dict.items()}\n    \n#     preds_flat = np.argmax(preds, axis=1).flatten()\n#     labels_flat = labels.flatten()\n\n#     for label in np.unique(labels_flat):\n#         y_preds = preds_flat[labels_flat==label]\n#         y_true = labels_flat[labels_flat==label]\n#         print(f'Class: {label_dict_inverse[label]}')\n#         print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training Loop","metadata":{}},{"cell_type":"code","source":"# import random\n\n# seed_val = 17\n# random.seed(seed_val)\n# np.random.seed(seed_val)\n# torch.manual_seed(seed_val)\n# torch.cuda.manual_seed_all(seed_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# model.to(device)\n\n# print(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def evaluate(dataloader_val):\n\n#     model.eval()\n    \n#     loss_val_total = 0\n#     predictions, true_vals = [], []\n    \n#     for batch in dataloader_val:\n        \n#         batch = tuple(b.to(device) for b in batch)\n        \n#         inputs = {'input_ids':      batch[0],\n#                   'attention_mask': batch[1],\n#                   'labels':         batch[2],\n#                  }\n\n#         with torch.no_grad():        \n#             outputs = model(**inputs)\n            \n#         loss = outputs[0]\n#         logits = outputs[1]\n#         loss_val_total += loss.item()\n\n#         logits = logits.detach().cpu().numpy()\n#         label_ids = inputs['labels'].cpu().numpy()\n#         predictions.append(logits)\n#         true_vals.append(label_ids)\n#     loss_val_avg = loss_val_total/len(dataloader_val) \n    \n#     predictions = np.concatenate(predictions, axis=0)\n#     true_vals = np.concatenate(true_vals, axis=0)\n            \n#     return loss_val_avg, predictions, true_vals","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for epoch in tqdm(range(1, epochs+1)):\n    \n#     model.train()\n    \n#     loss_train_total = 0\n\n#     progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)\n#     for batch in progress_bar:\n\n#         model.zero_grad()\n        \n#         batch = tuple(b.to(device) for b in batch)\n        \n#         inputs = {'input_ids':      batch[0],\n#                   'attention_mask': batch[1],\n#                   'labels':         batch[2],\n#                  }       \n\n#         outputs = model(**inputs)\n        \n#         loss = outputs[0]\n#         loss_train_total += loss.item()\n#         loss.backward()\n\n#         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n#         optimizer.step()\n#         scheduler.step()\n#         progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})\n         \n        \n#     torch.save(model.state_dict(), f'finetuned_BERT_epoch_{epoch}.model')\n        \n#     tqdm.write(f'\\nEpoch {epoch}')\n    \n#     loss_train_avg = loss_train_total/len(dataloader_train)            \n#     tqdm.write(f'Training loss: {loss_train_avg}')\n    \n#     val_loss, predictions, true_vals = evaluate(dataloader_validation)\n#     val_f1 = f1_score_func(predictions, true_vals)\n#     tqdm.write(f'Validation loss: {val_loss}')\n#     tqdm.write(f'F1 Score (Weighted): {val_f1}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading and Evaluating the Model","metadata":{}},{"cell_type":"code","source":"# model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n#                                                       num_labels=len(label_dict),\n#                                                       output_attentions=False,\n#                                                       output_hidden_states=False)\n\n# model.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_state_dict(torch.load('finetuned_BERT_epoch_1.model', map_location=torch.device('cpu')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# _, predictions, true_vals = evaluate(dataloader_validation)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# accuracy_per_class(predictions, true_vals)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}