{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import ","metadata":{}},{"cell_type":"code","source":"!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:13:31.653092Z","iopub.execute_input":"2023-06-20T11:13:31.653696Z","iopub.status.idle":"2023-06-20T11:13:44.956773Z","shell.execute_reply.started":"2023-06-20T11:13:31.653659Z","shell.execute_reply":"2023-06-20T11:13:44.955501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the libraries needed\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport transformers\nimport json\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import RobertaModel, RobertaTokenizer\nfrom sklearn.model_selection import train_test_split\nimport logging\nimport re\nimport string\nimport nltk\nfrom torch import cuda\nnltk.download('stopwords')\nnltk.download('punkt')\nnltk.download('wordnet')\n\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\n\nlogging.basicConfig(level=logging.ERROR)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T21:18:01.925470Z","iopub.execute_input":"2023-06-20T21:18:01.926457Z","iopub.status.idle":"2023-06-20T21:18:01.948393Z","shell.execute_reply.started":"2023-06-20T21:18:01.926404Z","shell.execute_reply":"2023-06-20T21:18:01.946913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stop_words = [ word for word in stop_words if word not in negation]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:14:09.724234Z","iopub.execute_input":"2023-06-20T11:14:09.724679Z","iopub.status.idle":"2023-06-20T11:14:10.257300Z","shell.execute_reply.started":"2023-06-20T11:14:09.724640Z","shell.execute_reply":"2023-06-20T11:14:10.255527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:52:08.919231Z","iopub.execute_input":"2023-06-20T20:52:08.919635Z","iopub.status.idle":"2023-06-20T20:52:08.989221Z","shell.execute_reply.started":"2023-06-20T20:52:08.919594Z","shell.execute_reply":"2023-06-20T20:52:08.987917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data explore","metadata":{}},{"cell_type":"code","source":"review_df = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:52:08.990625Z","iopub.execute_input":"2023-06-20T20:52:08.991629Z","iopub.status.idle":"2023-06-20T20:52:28.145370Z","shell.execute_reply.started":"2023-06-20T20:52:08.991583Z","shell.execute_reply":"2023-06-20T20:52:28.144272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:52:28.148504Z","iopub.execute_input":"2023-06-20T20:52:28.148971Z","iopub.status.idle":"2023-06-20T20:52:28.173984Z","shell.execute_reply.started":"2023-06-20T20:52:28.148921Z","shell.execute_reply":"2023-06-20T20:52:28.173037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop the user_id, Book_id, review_id, date_added, date_updated, read_at, and started_at we don't need them in our pipline","metadata":{}},{"cell_type":"code","source":"# Drop Dubplicate rows\nreview_df.drop_duplicates(subset=['review_text'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:53:10.550182Z","iopub.execute_input":"2023-06-20T20:53:10.550782Z","iopub.status.idle":"2023-06-20T20:53:12.839803Z","shell.execute_reply.started":"2023-06-20T20:53:10.550741Z","shell.execute_reply":"2023-06-20T20:53:12.838706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.drop([\"user_id\", \"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:53:12.842060Z","iopub.execute_input":"2023-06-20T20:53:12.842478Z","iopub.status.idle":"2023-06-20T20:53:13.414429Z","shell.execute_reply.started":"2023-06-20T20:53:12.842438Z","shell.execute_reply":"2023-06-20T20:53:13.412671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:52:30.687010Z","iopub.execute_input":"2023-06-20T20:52:30.687617Z","iopub.status.idle":"2023-06-20T20:52:30.703616Z","shell.execute_reply.started":"2023-06-20T20:52:30.687574Z","shell.execute_reply":"2023-06-20T20:52:30.702496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show the rating distribustion ","metadata":{}},{"cell_type":"code","source":"review_df.rating.value_counts().plot(kind=\"bar\", xlabel=\"rates\", ylabel=\"Frequency\", title=\"Rates Frequency\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:29.382196Z","iopub.execute_input":"2023-03-31T20:06:29.383032Z","iopub.status.idle":"2023-03-31T20:06:29.646669Z","shell.execute_reply.started":"2023-03-31T20:06:29.382995Z","shell.execute_reply":"2023-03-31T20:06:29.645636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<strong> from this figure we can see that the rates 0 and 1 is less frequency rather than other rates  </strong>","metadata":{}},{"cell_type":"markdown","source":"## the correlation between votes and n_comments and rates|","metadata":{}},{"cell_type":"code","source":"plt.title(\"n_comments dis\")\nsns.histplot(data = review_df, x= 'n_comments', bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:29.648241Z","iopub.execute_input":"2023-03-31T20:06:29.649341Z","iopub.status.idle":"2023-03-31T20:06:30.216776Z","shell.execute_reply.started":"2023-03-31T20:06:29.649300Z","shell.execute_reply":"2023-03-31T20:06:30.215719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.n_comments.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:30.218125Z","iopub.execute_input":"2023-03-31T20:06:30.220161Z","iopub.status.idle":"2023-03-31T20:06:30.247977Z","shell.execute_reply.started":"2023-03-31T20:06:30.220119Z","shell.execute_reply":"2023-03-31T20:06:30.246755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"n_votes dis\")\nsns.histplot(data = review_df, x= 'n_votes', bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:30.249550Z","iopub.execute_input":"2023-03-31T20:06:30.252045Z","iopub.status.idle":"2023-03-31T20:06:30.774760Z","shell.execute_reply.started":"2023-03-31T20:06:30.252004Z","shell.execute_reply":"2023-03-31T20:06:30.773660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.n_votes.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:30.778251Z","iopub.execute_input":"2023-03-31T20:06:30.778543Z","iopub.status.idle":"2023-03-31T20:06:30.809015Z","shell.execute_reply.started":"2023-03-31T20:06:30.778516Z","shell.execute_reply":"2023-03-31T20:06:30.806987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**From this results n_votes and n_comments will not usefull for us**","metadata":{}},{"cell_type":"code","source":"review_df.drop([\"n_comments\", \"n_votes\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T13:22:29.415332Z","iopub.execute_input":"2023-04-01T13:22:29.415698Z","iopub.status.idle":"2023-04-01T13:22:29.469745Z","shell.execute_reply.started":"2023-04-01T13:22:29.415662Z","shell.execute_reply":"2023-04-01T13:22:29.468744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Take a Sample from the original data","metadata":{}},{"cell_type":"code","source":"# review_df = review_df.sample(int(len(review_df) * 0.3), random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:06:30.869639Z","iopub.execute_input":"2023-03-31T20:06:30.870032Z","iopub.status.idle":"2023-03-31T20:06:30.931962Z","shell.execute_reply.started":"2023-03-31T20:06:30.869995Z","shell.execute_reply":"2023-03-31T20:06:30.930934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.rating.value_counts().plot(kind=\"bar\", xlabel=\"rates\", ylabel=\"Frequency\", title=\"Rates Frequency\")","metadata":{"execution":{"iopub.status.busy":"2023-03-31T20:07:16.690966Z","iopub.execute_input":"2023-03-31T20:07:16.691640Z","iopub.status.idle":"2023-03-31T20:07:16.961226Z","shell.execute_reply.started":"2023-03-31T20:07:16.691601Z","shell.execute_reply":"2023-03-31T20:07:16.953259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" <strong >fine tune code is from https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb </strong>\n ","metadata":{}},{"cell_type":"markdown","source":"# Text preprocessing","metadata":{}},{"cell_type":"code","source":"# Defining some key variables that will be used later on in the training\nMAX_LEN = 256\nTRAIN_BATCH_SIZE = 8\nVALID_BATCH_SIZE = 4\n# EPOCHS = 1\nLEARNING_RATE = 1e-05\ntokenizer = RobertaTokenizer.from_pretrained('roberta-base', truncation=True, do_lower_case=True)\n# tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased',  truncation=True, do_lower_case=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:54:18.015488Z","iopub.execute_input":"2023-06-20T20:54:18.016459Z","iopub.status.idle":"2023-06-20T20:54:21.147062Z","shell.execute_reply.started":"2023-06-20T20:54:18.016402Z","shell.execute_reply":"2023-06-20T20:54:21.146011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words('english'))\nnegation_words = [\"didn\", \"weren\", \"doesn't\",\" hasn't\", \"couldn\",\n \"mustn't\", \"isn\", \"hadn't\", \"isn't\", \"wasn't\", \"mightn't\",\n \"couldn't\", \"needn't\", \"haven't\", \"shan't\", \"wouldn't\",\n \"not\", \"no\", \"never\", \"none\", \"nobody\", \"nowhere\", \"nothing\",\"isn'\", \"'t\",\n \"shouldn't\", \"aren't\", \"didn't\", \"didn'\", \"don't\", \"hadn't\", \"won't\", \"wasn\", \"hadn\"]\nstop_words = [ word for word in stop_words if word not in negation_words]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:54:31.283733Z","iopub.execute_input":"2023-06-20T20:54:31.284153Z","iopub.status.idle":"2023-06-20T20:54:31.295182Z","shell.execute_reply.started":"2023-06-20T20:54:31.284117Z","shell.execute_reply":"2023-06-20T20:54:31.293951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SentimentData(Dataset):\n    def __init__(self, dataframe_input, tokenizer, max_len, stop_wrods, negation_words, data_type=\"train\"):\n        self.tokenizer = tokenizer\n        self.data_type = data_type\n        self.data = dataframe_input\n        self.text = dataframe_input.review_text.values\n        self.stop_wrods = stop_wrods\n        self.negation_words = negation_words\n\n        if data_type != \"train\":\n            self.review_id = self.data.review_id.values\n\n        if data_type == \"train\":\n            self.targets = self.data.rating\n        self.max_len = max_len\n\n    def __len__(self):\n        return len(self.text)\n    \n    def preprocess_text(self, text):\n        # Remove unnecessary white spaces\n        text = \" \".join(text.split())\n\n        # Remove HTML tags\n        text = re.sub('<[^<]+?>', '', text)\n\n        # Remove URLs\n        text = re.sub(r'http\\S+', '', text)\n\n        # Convert to lowercase\n        text = text.lower()\n\n        # Remove punctuation\n        text = text.translate(str.maketrans('', '', string.punctuation))\n         \n        text = re.sub(r'[^a-zA-Z\\s]', '', text)\n        \n        splited_text = text.split()\n        cleaned_text = []\n        for word in splited_text:\n            if word not in self.stop_wrods:\n                cleaned_text.append(word)\n            elif word in self.negation_words:\n                cleaned_text.append(\"not\")\n        text  = \" \".join(cleaned_text)\n\n\n#         Tokenize the text\n#         tokens = tokenizer.tokenize(text)\n\n#         Remove stopwords\n#         c\n#         tokens = [word for word in tokens if not word in stop_words]\n\n#         Lemmatize the tokens\n#         lemmatizer = WordNetLemmatizer()\n#         tokens = [lemmatizer.lemmatize(word) for word in tokens]\n\n#         Join the tokens back into a string\n#         preprocessed_text = ' '.join(tokens)\n\n        return text\n\n    def __getitem__(self, index):\n        text = str(self.text[index])\n        text = self.preprocess_text(text)\n        \n               \n        # Replace Negation words with [Not] Token\n#         for i in range(len(tokens)):\n#             sub_tokens = tokens[i].split('Ġ')\n#             for j in range(len(sub_tokens)):\n#                 if sub_tokens[j] in negation_words:\n#                     sub_tokens[j] = '[NOT]'\n#             tokens[i] = 'Ġ'.join(sub_tokens)\n\n\n        inputs = self.tokenizer.encode_plus(\n            text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            pad_to_max_length=True,\n            return_token_type_ids=True\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        token_type_ids = inputs[\"token_type_ids\"]\n        \n        if self.data_type == \"train\":\n             return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'targets':  torch.tensor(self.targets[index], dtype=torch.float)\n\n        }\n\n        else:\n             return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'review_id' : self.review_id[index]\n                }","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:55:31.369033Z","iopub.execute_input":"2023-06-20T20:55:31.369681Z","iopub.status.idle":"2023-06-20T20:55:31.395658Z","shell.execute_reply.started":"2023-06-20T20:55:31.369634Z","shell.execute_reply":"2023-06-20T20:55:31.394367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Test Split","metadata":{"execution":{"iopub.status.busy":"2023-03-29T21:36:48.168381Z","iopub.execute_input":"2023-03-29T21:36:48.169703Z","iopub.status.idle":"2023-03-29T21:36:48.177088Z","shell.execute_reply.started":"2023-03-29T21:36:48.169644Z","shell.execute_reply":"2023-03-29T21:36:48.176334Z"}}},{"cell_type":"code","source":"X_train, X_test, _, _ = train_test_split(review_df, review_df, test_size=0.1, random_state=42)\nX_valid, X_test, _, _ = train_test_split(X_test, X_test, test_size=0.5, random_state=42)\n\nX_train.reset_index(inplace=True, drop=True)\nX_test.reset_index(inplace=True, drop=True)\nX_valid.reset_index(inplace=True, drop=True)\n\nprint(\"FULL Dataset: {}\".format(review_df.shape))\nprint(\"TRAIN Dataset: {}\".format(X_train.shape))\nprint(\"Valid Dataset: {}\".format(X_valid.shape))\nprint(\"TEST Dataset: {}\".format(X_test.shape))\n\ntraining_set = SentimentData(X_train, tokenizer, MAX_LEN, stop_words, negation_words )\nvalidation_set = SentimentData(X_valid, tokenizer, MAX_LEN, stop_words, negation_words)\ntesting_set = SentimentData(X_test, tokenizer, MAX_LEN, stop_words, negation_words)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:55:36.367384Z","iopub.execute_input":"2023-06-20T20:55:36.368115Z","iopub.status.idle":"2023-06-20T20:55:36.792066Z","shell.execute_reply.started":"2023-06-20T20:55:36.368048Z","shell.execute_reply":"2023-06-20T20:55:36.790833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_params = {'batch_size': TRAIN_BATCH_SIZE,\n                'shuffle': True,\n                'num_workers': 2,\n                'pin_memory':True\n                }\n\ntest_params = {'batch_size': VALID_BATCH_SIZE,\n                'shuffle': True,\n                'num_workers': 2,\n               'pin_memory':True\n                }\n\ntraining_loader = DataLoader(training_set, **train_params)\nvlidation_loader = DataLoader(validation_set, **train_params)\n\ntesting_loader = DataLoader(testing_set, **test_params)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:55:40.289043Z","iopub.execute_input":"2023-06-20T20:55:40.289794Z","iopub.status.idle":"2023-06-20T20:55:40.297381Z","shell.execute_reply.started":"2023-06-20T20:55:40.289754Z","shell.execute_reply":"2023-06-20T20:55:40.295825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"class RobertaClass(torch.nn.Module):\n    def __init__(self):\n        super(RobertaClass, self).__init__()\n        self.l1 = RobertaModel.from_pretrained(\"roberta-base\")\n        self.pre_classifier = torch.nn.Linear(768, 768)\n        self.dropout = torch.nn.Dropout(0.3)\n        self.classifier = torch.nn.Linear(768, 6)\n\n    def forward(self, input_ids, attention_mask, token_type_ids):\n        output_1 = self.l1(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n        hidden_state = output_1[0]\n        pooler = hidden_state[:, 0]\n        pooler = self.pre_classifier(pooler)\n        pooler = torch.nn.ReLU()(pooler)\n        pooler = self.dropout(pooler)\n        output = self.classifier(pooler)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:01.463920Z","iopub.execute_input":"2023-06-20T20:57:01.464377Z","iopub.status.idle":"2023-06-20T20:57:01.473743Z","shell.execute_reply.started":"2023-06-20T20:57:01.464340Z","shell.execute_reply":"2023-06-20T20:57:01.472523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RobertaClass()\nmodel.to(device)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = X_train.rating.value_counts()\nclass0 = counts[0]\nclass1 = counts[1]\nclass2 = counts[2]\nclass3 = counts[3]\nclass4 = counts[4]\nclass5 = counts[5]\n\ntotal = class0 + class1 + class1 + class2 + class3 + class4 + class5\n\nweight_for_0 = (1 / class0) * (total / 2.0)\nweight_for_1 = (1 / class1) * (total / 2.0)\nweight_for_2 = (1 / class2) * (total / 2.0)\nweight_for_3 = (1 / class3) * (total / 2.0)\nweight_for_4 = (1 / class4) * (total / 2.0)\nweight_for_5 = (1 / class5) * (total / 2.0)\n\nclass_w = [weight_for_0, weight_for_1, weight_for_2, weight_for_3, weight_for_4, weight_for_5]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:44.532059Z","iopub.execute_input":"2023-06-20T20:57:44.532855Z","iopub.status.idle":"2023-06-20T20:57:44.550473Z","shell.execute_reply.started":"2023-06-20T20:57:44.532817Z","shell.execute_reply":"2023-06-20T20:57:44.549208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the loss function and optimizer\nloss_function = torch.nn.CrossEntropyLoss(weight =  torch.FloatTensor(class_w).to(device))\noptimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:45.159613Z","iopub.execute_input":"2023-06-20T20:57:45.160003Z","iopub.status.idle":"2023-06-20T20:57:45.169536Z","shell.execute_reply.started":"2023-06-20T20:57:45.159968Z","shell.execute_reply":"2023-06-20T20:57:45.168367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calcuate_accuracy(preds, targets):\n    n_correct = (preds==targets).sum().item()\n    return n_correct","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:45.726201Z","iopub.execute_input":"2023-06-20T20:57:45.726983Z","iopub.status.idle":"2023-06-20T20:57:45.735208Z","shell.execute_reply.started":"2023-06-20T20:57:45.726945Z","shell.execute_reply":"2023-06-20T20:57:45.734223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the training function on the 80% of the dataset for tuning the distilbert model\ndef train(epoch):\n    tr_loss = 0\n    n_correct = 0\n    nb_tr_steps = 0\n    nb_tr_examples = 0\n    model.train()\n    for _,data in tqdm(enumerate(training_loader, 0)):\n        ids = data['ids'].to(device, dtype = torch.long)\n        mask = data['mask'].to(device, dtype = torch.long)\n        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n        targets = data['targets'].to(device, dtype = torch.long)\n\n        outputs = model(ids, mask, token_type_ids)\n        loss = loss_function(outputs, targets)\n        tr_loss += loss.item()\n        big_val, big_idx = torch.max(outputs.data, dim=1)\n        n_correct += calcuate_accuracy(big_idx, targets)\n\n        nb_tr_steps += 1\n        nb_tr_examples+=targets.size(0)\n        \n        if _%5000==0:\n            loss_step = tr_loss/nb_tr_steps\n            accu_step = (n_correct*100)/nb_tr_examples \n            print(f\"Training Loss per 5000 steps: {loss_step}\")\n            print(f\"Training Accuracy per 5000 steps: {accu_step}\")\n\n        optimizer.zero_grad()\n        loss.backward()\n        # # When using GPU\n        optimizer.step()\n\n    print(f'The Total Accuracy for Epoch {epoch}: {(n_correct*100)/nb_tr_examples}')\n    epoch_loss = tr_loss/nb_tr_steps\n    epoch_accu = (n_correct*100)/nb_tr_examples\n    print(f\"Training Loss Epoch: {epoch_loss}\")\n    print(f\"Training Accuracy Epoch: {epoch_accu}\")\n\n    return ","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:49.559852Z","iopub.execute_input":"2023-06-20T20:57:49.560255Z","iopub.status.idle":"2023-06-20T20:57:49.572649Z","shell.execute_reply.started":"2023-06-20T20:57:49.560221Z","shell.execute_reply":"2023-06-20T20:57:49.571555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1\nfor epoch in range(EPOCHS):\n    train(epoch)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T20:57:54.172471Z","iopub.execute_input":"2023-06-20T20:57:54.173066Z","iopub.status.idle":"2023-06-20T20:57:54.785798Z","shell.execute_reply.started":"2023-06-20T20:57:54.173028Z","shell.execute_reply":"2023-06-20T20:57:54.784131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer.save_pretrained(\"./tokinizer3\")\n\n# print('All files saved')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submmetion","metadata":{}},{"cell_type":"code","source":"new_tokenizer = RobertaTokenizer.from_pretrained('/kaggle/working/tokinizer2')\nmodel.load_state_dict(torch.load('/kaggle/working/model_RoBERTa2.pt'))\nmodel.to(\"cpu\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quantized_model = torch.quantization.quantize_dynamic(\n    model, {torch.nn.Linear}, dtype=torch.qint8\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:30:04.206563Z","iopub.execute_input":"2023-06-20T11:30:04.207490Z","iopub.status.idle":"2023-06-20T11:30:05.833193Z","shell.execute_reply.started":"2023-06-20T11:30:04.207449Z","shell.execute_reply":"2023-06-20T11:30:05.831994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quantized_model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv\")\nfinal_test.drop([\"user_id\", \"book_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\", \"n_comments\", \"n_votes\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:17:00.409225Z","iopub.execute_input":"2023-06-20T11:17:00.409850Z","iopub.status.idle":"2023-06-20T11:17:06.424580Z","shell.execute_reply.started":"2023-06-20T11:17:00.409809Z","shell.execute_reply":"2023-06-20T11:17:06.423428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.drop(final_test[1:].index.values, inplace =True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:16:51.227443Z","iopub.execute_input":"2023-06-20T11:16:51.228124Z","iopub.status.idle":"2023-06-20T11:16:51.313662Z","shell.execute_reply.started":"2023-06-20T11:16:51.228079Z","shell.execute_reply":"2023-06-20T11:16:51.312603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.loc[0,\"review_text\"]= \"I don't hate\" ","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:16:51.461670Z","iopub.execute_input":"2023-06-20T11:16:51.462166Z","iopub.status.idle":"2023-06-20T11:16:51.469833Z","shell.execute_reply.started":"2023-06-20T11:16:51.462129Z","shell.execute_reply":"2023-06-20T11:16:51.468613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:17:06.426825Z","iopub.execute_input":"2023-06-20T11:17:06.427263Z","iopub.status.idle":"2023-06-20T11:17:06.440891Z","shell.execute_reply.started":"2023-06-20T11:17:06.427211Z","shell.execute_reply":"2023-06-20T11:17:06.439575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_set = SentimentData(final_test, new_tokenizer, MAX_LEN, stop_words, negation_words, \"test\")\ntest_params = {'batch_size': 32,\n                'shuffle': False,\n                'num_workers': 2,\n               'pin_memory':True\n                }\n\nfinal_test_set = DataLoader(final_test_set, **test_params)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:18:53.175329Z","iopub.execute_input":"2023-06-20T11:18:53.175719Z","iopub.status.idle":"2023-06-20T11:18:53.183159Z","shell.execute_reply.started":"2023-06-20T11:18:53.175684Z","shell.execute_reply":"2023-06-20T11:18:53.181896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:18:53.862165Z","iopub.execute_input":"2023-06-20T11:18:53.863346Z","iopub.status.idle":"2023-06-20T11:18:53.869123Z","shell.execute_reply.started":"2023-06-20T11:18:53.863301Z","shell.execute_reply":"2023-06-20T11:18:53.867895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def final_submmit(model, testing_loader):\n    device = 'cpu'\n    prediction_ids = []\n    preds = []\n    model.eval()\n    \n    with torch.no_grad():\n        for _, data in tqdm(enumerate(testing_loader, 0)):\n            ids = data['ids'].to(device, dtype = torch.long)\n            mask = data['mask'].to(device, dtype = torch.long)\n            token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)\n            review_id = data[\"review_id\"]\n            outputs = model(ids, mask, token_type_ids)\n            softmax_output = F.softmax(outputs, dim=1)\n            predicted_classes = torch.argmax(softmax_output, dim=1)\n            prediction_ids.extend(review_id)\n            preds.extend(predicted_classes.cpu().numpy())\n\n        df = pd.DataFrame({'review_id': prediction_ids , \"rating\": preds })\n        return df\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:30:22.481834Z","iopub.execute_input":"2023-06-20T11:30:22.482830Z","iopub.status.idle":"2023-06-20T11:30:22.493930Z","shell.execute_reply.started":"2023-06-20T11:30:22.482764Z","shell.execute_reply":"2023-06-20T11:30:22.492696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit = final_submmit(quantized_model, final_test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:20:20.239787Z","iopub.status.idle":"2023-06-20T11:20:20.240676Z","shell.execute_reply.started":"2023-06-20T11:20:20.240407Z","shell.execute_reply":"2023-06-20T11:20:20.240435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit.to_csv(\"submmesion2.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T13:22:51.648727Z","iopub.execute_input":"2023-06-03T13:22:51.649401Z","iopub.status.idle":"2023-06-03T13:22:51.656688Z","shell.execute_reply.started":"2023-06-03T13:22:51.649356Z","shell.execute_reply":"2023-06-03T13:22:51.655344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T13:22:20.827493Z","iopub.execute_input":"2023-06-03T13:22:20.828525Z","iopub.status.idle":"2023-06-03T13:22:21.569006Z","shell.execute_reply.started":"2023-06-03T13:22:20.828481Z","shell.execute_reply":"2023-06-03T13:22:21.567868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"0.88 0.4 0.2 0.2 0.2 0.2","metadata":{"execution":{"iopub.status.busy":"2023-03-31T01:09:19.847961Z","iopub.execute_input":"2023-03-31T01:09:19.848501Z","iopub.status.idle":"2023-03-31T01:09:19.870084Z","shell.execute_reply.started":"2023-03-31T01:09:19.848451Z","shell.execute_reply":"2023-03-31T01:09:19.869091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words('english'))\nnegation_words = [\"didn\", \"weren\", \"doesn't\",\" hasn't\", \"couldn\",\n \"mustn't\", \"isn\", \"hadn't\", \"isn't\", \"wasn't\", \"mightn't\",\n \"couldn't\", \"needn't\", \"haven't\", \"shan't\", \"wouldn't\",\n \"not\", \"no\", \"never\", \"none\", \"nobody\", \"nowhere\", \"nothing\",\"isn'\", \"'t\",\n \"shouldn't\", \"aren't\", \"didn't\", \"didn'\", \"don't\", \"hadn't\", \"won't\", \"wasn\", \"hadn\"]\nstop_words = [ word for word in stop_words if word not in negation_words]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Senti_Toknizer():\n    def __init__(self, tokenizer, stop_wrods, negation_words, max_len=256):\n        self.tokenizer = tokenizer\n        self.stop_wrods = stop_wrods\n        self.negation_words = negation_words\n        self.max_len = max_len\n        \n    def preprocess_text(self, text):\n        # Remove unnecessary white spaces\n        text = \" \".join(text.split())\n\n        # Remove HTML tags\n        text = re.sub('<[^<]+?>', '', text)\n\n        # Remove URLs\n        text = re.sub(r'http\\S+', '', text)\n\n        # Convert to lowercase\n        text = text.lower()\n\n        # Remove punctuation\n        text = text.translate(str.maketrans('', '', string.punctuation))\n         \n        text = re.sub(r'[^a-zA-Z\\s]', '', text)\n\n        splited_text = text.split()\n        cleaned_text = []\n        for word in splited_text:\n            if word not in self.stop_wrods:\n                cleaned_text.append(word)\n            elif word in self.negation_words:\n                cleaned_text.append(\"not\")\n        text  = \" \".join(cleaned_text)\n        return text\n\n    def toknize(self, text):\n        text = self.preprocess_text(text)\n        inputs = self.tokenizer.encode_plus(\n            text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            pad_to_max_length=True,\n            return_token_type_ids=True\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        token_type_ids = inputs[\"token_type_ids\"]\n        \n        return {\n        'ids': torch.tensor(ids, dtype=torch.long),\n        'mask': torch.tensor(mask, dtype=torch.long),\n        'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)}\n","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:27:16.841079Z","iopub.execute_input":"2023-06-03T15:27:16.842229Z","iopub.status.idle":"2023-06-03T15:27:16.858732Z","shell.execute_reply.started":"2023-06-03T15:27:16.842184Z","shell.execute_reply":"2023-06-03T15:27:16.857397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = Senti_Toknizer( new_tokenizer, stop_words, negation_words,)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:27:17.377067Z","iopub.execute_input":"2023-06-03T15:27:17.378027Z","iopub.status.idle":"2023-06-03T15:27:17.383047Z","shell.execute_reply.started":"2023-06-03T15:27:17.377987Z","shell.execute_reply":"2023-06-03T15:27:17.381854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = g.toknize(\"this book is not so good but the matrial of the paper is good over all this book is well made\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:47:04.522708Z","iopub.execute_input":"2023-06-03T15:47:04.523412Z","iopub.status.idle":"2023-06-03T15:47:04.529922Z","shell.execute_reply.started":"2023-06-03T15:47:04.523372Z","shell.execute_reply":"2023-06-03T15:47:04.528595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    \n    ids = y['ids'].to(device, dtype = torch.long).unsqueeze(0)\n    mask = y['mask'].to(device, dtype = torch.long).unsqueeze(0)\n    token_type_ids = y['token_type_ids'].to(device, dtype=torch.long).unsqueeze(0)  \n    outputs = model(ids, mask, token_type_ids)\n    \n    softmax_output = F.softmax(outputs, dim=1)\n\n    N_propa = torch.sum(softmax_output[0,0:3]) \n    P_propa = torch.sum(softmax_output[0,3:])\n    \n    print (f\"Positive probability {P_propa}, Negative Propability {N_propa}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:47:04.843702Z","iopub.execute_input":"2023-06-03T15:47:04.844643Z","iopub.status.idle":"2023-06-03T15:47:04.871392Z","shell.execute_reply.started":"2023-06-03T15:47:04.844591Z","shell.execute_reply":"2023-06-03T15:47:04.870002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:30.194256Z","iopub.execute_input":"2023-06-03T15:45:30.195298Z","iopub.status.idle":"2023-06-03T15:45:30.205650Z","shell.execute_reply.started":"2023-06-03T15:45:30.195245Z","shell.execute_reply":"2023-06-03T15:45:30.204215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output[0,0:3]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:43.599451Z","iopub.execute_input":"2023-06-03T15:45:43.599896Z","iopub.status.idle":"2023-06-03T15:45:43.610050Z","shell.execute_reply.started":"2023-06-03T15:45:43.599858Z","shell.execute_reply":"2023-06-03T15:45:43.608691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output[0,3:]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:46:07.410880Z","iopub.execute_input":"2023-06-03T15:46:07.411274Z","iopub.status.idle":"2023-06-03T15:46:07.423226Z","shell.execute_reply.started":"2023-06-03T15:46:07.411237Z","shell.execute_reply":"2023-06-03T15:46:07.421734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:59.406351Z","iopub.execute_input":"2023-06-03T15:45:59.407354Z","iopub.status.idle":"2023-06-03T15:45:59.417059Z","shell.execute_reply.started":"2023-06-03T15:45:59.407313Z","shell.execute_reply":"2023-06-03T15:45:59.415810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(quantized_model, 'qun_model_RoBERTa3w.pth')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:55.126088Z","iopub.execute_input":"2023-06-20T11:58:55.127133Z","iopub.status.idle":"2023-06-20T11:58:56.139478Z","shell.execute_reply.started":"2023-06-20T11:58:55.127079Z","shell.execute_reply":"2023-06-20T11:58:56.138134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = torch.load('/kaggle/working/qun_model_RoBERTa3w.pth')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:57.926845Z","iopub.execute_input":"2023-06-20T11:58:57.927882Z","iopub.status.idle":"2023-06-20T11:58:58.690236Z","shell.execute_reply.started":"2023-06-20T11:58:57.927827Z","shell.execute_reply":"2023-06-20T11:58:58.689094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:59.725125Z","iopub.execute_input":"2023-06-20T11:58:59.726378Z","iopub.status.idle":"2023-06-20T11:59:00.178388Z","shell.execute_reply.started":"2023-06-20T11:58:59.726332Z","shell.execute_reply":"2023-06-20T11:59:00.177070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}