{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import ","metadata":{}},{"cell_type":"code","source":"!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:13:31.653092Z","iopub.execute_input":"2023-06-20T11:13:31.653696Z","iopub.status.idle":"2023-06-20T11:13:44.956773Z","shell.execute_reply.started":"2023-06-20T11:13:31.653659Z","shell.execute_reply":"2023-06-20T11:13:44.955501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport subprocess\nfrom IPython.display import FileLink, display\n","metadata":{"execution":{"iopub.status.busy":"2023-06-24T10:41:54.657563Z","iopub.execute_input":"2023-06-24T10:41:54.658024Z","iopub.status.idle":"2023-06-24T10:41:54.666561Z","shell.execute_reply.started":"2023-06-24T10:41:54.657987Z","shell.execute_reply":"2023-06-24T10:41:54.665047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    display(FileLink(f'{download_file_name}.zip'))","metadata":{"execution":{"iopub.status.busy":"2023-06-24T10:41:55.635903Z","iopub.execute_input":"2023-06-24T10:41:55.636250Z","iopub.status.idle":"2023-06-24T10:41:55.643716Z","shell.execute_reply.started":"2023-06-24T10:41:55.636222Z","shell.execute_reply":"2023-06-24T10:41:55.642647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the libraries needed\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport transformers\nimport json\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import RobertaModel, RobertaTokenizer\nfrom transformers import DistilBertTokenizer, DistilBertForSequenceClassification, DistilBertModel\n\nfrom sklearn.model_selection import train_test_split\nimport logging\nimport re\nimport string\nimport nltk\nfrom torch import cuda\nnltk.download('stopwords')\nnltk.download('punkt')\nnltk.download('wordnet')\n\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\n\nlogging.basicConfig(level=logging.ERROR)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:11.759555Z","iopub.execute_input":"2023-06-21T20:07:11.759849Z","iopub.status.idle":"2023-06-21T20:07:27.926401Z","shell.execute_reply.started":"2023-06-21T20:07:11.759822Z","shell.execute_reply":"2023-06-21T20:07:27.925463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stop_words = [ word for word in stop_words if word not in negation]","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:27.928473Z","iopub.execute_input":"2023-06-21T20:07:27.928820Z","iopub.status.idle":"2023-06-21T20:07:27.933480Z","shell.execute_reply.started":"2023-06-21T20:07:27.928788Z","shell.execute_reply":"2023-06-21T20:07:27.932478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:27.935023Z","iopub.execute_input":"2023-06-21T20:07:27.935358Z","iopub.status.idle":"2023-06-21T20:07:27.968759Z","shell.execute_reply.started":"2023-06-21T20:07:27.935326Z","shell.execute_reply":"2023-06-21T20:07:27.967799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data explore","metadata":{}},{"cell_type":"code","source":"review_df = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:27.973210Z","iopub.execute_input":"2023-06-21T20:07:27.975067Z","iopub.status.idle":"2023-06-21T20:07:50.559087Z","shell.execute_reply.started":"2023-06-21T20:07:27.975041Z","shell.execute_reply":"2023-06-21T20:07:50.558155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:50.560414Z","iopub.execute_input":"2023-06-21T20:07:50.561040Z","iopub.status.idle":"2023-06-21T20:07:50.584900Z","shell.execute_reply.started":"2023-06-21T20:07:50.561005Z","shell.execute_reply":"2023-06-21T20:07:50.583981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop the user_id, Book_id, review_id, date_added, date_updated, read_at, and started_at we don't need them in our pipline","metadata":{}},{"cell_type":"code","source":"# Drop Dubplicate rows\nreview_df.drop_duplicates(subset=['review_text'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:50.586267Z","iopub.execute_input":"2023-06-21T20:07:50.586701Z","iopub.status.idle":"2023-06-21T20:07:51.399553Z","shell.execute_reply.started":"2023-06-21T20:07:50.586665Z","shell.execute_reply":"2023-06-21T20:07:51.398591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.drop([\"user_id\", \"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:51.400918Z","iopub.execute_input":"2023-06-21T20:07:51.401531Z","iopub.status.idle":"2023-06-21T20:07:51.484896Z","shell.execute_reply.started":"2023-06-21T20:07:51.401497Z","shell.execute_reply":"2023-06-21T20:07:51.483926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:51.486532Z","iopub.execute_input":"2023-06-21T20:07:51.487194Z","iopub.status.idle":"2023-06-21T20:07:51.506548Z","shell.execute_reply.started":"2023-06-21T20:07:51.487114Z","shell.execute_reply":"2023-06-21T20:07:51.505554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show the rating distribustion ","metadata":{}},{"cell_type":"code","source":"review_df.rating.value_counts().plot(kind=\"bar\", xlabel=\"rates\", ylabel=\"Frequency\", title=\"Rates Frequency\")","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:51.508107Z","iopub.execute_input":"2023-06-21T20:07:51.508450Z","iopub.status.idle":"2023-06-21T20:07:51.833529Z","shell.execute_reply.started":"2023-06-21T20:07:51.508419Z","shell.execute_reply":"2023-06-21T20:07:51.832626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<strong> from this figure we can see that the rates 0 and 1 is less frequency rather than other rates  </strong>","metadata":{}},{"cell_type":"markdown","source":"## the correlation between votes and n_comments and rates|","metadata":{}},{"cell_type":"code","source":"plt.title(\"n_comments dis\")\nsns.histplot(data = review_df, x= 'n_comments', bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:51.838228Z","iopub.execute_input":"2023-06-21T20:07:51.838503Z","iopub.status.idle":"2023-06-21T20:07:52.566844Z","shell.execute_reply.started":"2023-06-21T20:07:51.838478Z","shell.execute_reply":"2023-06-21T20:07:52.565926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.n_comments.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:52.568268Z","iopub.execute_input":"2023-06-21T20:07:52.568597Z","iopub.status.idle":"2023-06-21T20:07:52.597098Z","shell.execute_reply.started":"2023-06-21T20:07:52.568565Z","shell.execute_reply":"2023-06-21T20:07:52.595814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"n_votes dis\")\nsns.histplot(data = review_df, x= 'n_votes', bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:52.598699Z","iopub.execute_input":"2023-06-21T20:07:52.599559Z","iopub.status.idle":"2023-06-21T20:07:53.333871Z","shell.execute_reply.started":"2023-06-21T20:07:52.599532Z","shell.execute_reply":"2023-06-21T20:07:53.332874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.n_votes.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:53.335084Z","iopub.execute_input":"2023-06-21T20:07:53.335422Z","iopub.status.idle":"2023-06-21T20:07:53.365874Z","shell.execute_reply.started":"2023-06-21T20:07:53.335388Z","shell.execute_reply":"2023-06-21T20:07:53.364849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**From this results n_votes and n_comments will not usefull for us**","metadata":{}},{"cell_type":"code","source":"review_df.drop([\"n_comments\", \"n_votes\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:53.367273Z","iopub.execute_input":"2023-06-21T20:07:53.367607Z","iopub.status.idle":"2023-06-21T20:07:53.422714Z","shell.execute_reply.started":"2023-06-21T20:07:53.367574Z","shell.execute_reply":"2023-06-21T20:07:53.421703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Take a Sample from the original data","metadata":{}},{"cell_type":"code","source":"# review_df = review_df.sample(int(len(review_df) * 0.3), random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:53.423908Z","iopub.execute_input":"2023-06-21T20:07:53.424729Z","iopub.status.idle":"2023-06-21T20:07:53.429140Z","shell.execute_reply.started":"2023-06-21T20:07:53.424693Z","shell.execute_reply":"2023-06-21T20:07:53.427789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"review_df.rating.value_counts().plot(kind=\"bar\", xlabel=\"rates\", ylabel=\"Frequency\", title=\"Rates Frequency\")","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:53.430490Z","iopub.execute_input":"2023-06-21T20:07:53.431514Z","iopub.status.idle":"2023-06-21T20:07:53.765725Z","shell.execute_reply.started":"2023-06-21T20:07:53.431482Z","shell.execute_reply":"2023-06-21T20:07:53.764835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" <strong >fine tune code is from https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb </strong>\n ","metadata":{}},{"cell_type":"markdown","source":"# Text preprocessing","metadata":{}},{"cell_type":"code","source":"# Defining some key variables that will be used later on in the training\nMAX_LEN = 256\nTRAIN_BATCH_SIZE = 8\nVALID_BATCH_SIZE = 4\n# EPOCHS = 1\nLEARNING_RATE = 1e-05\n# tokenizer = RobertaTokenizer.from_pretrained('roberta-base', truncation=True, do_lower_case=True)\ntokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased',  truncation=True, do_lower_case=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:53.767138Z","iopub.execute_input":"2023-06-21T20:07:53.767758Z","iopub.status.idle":"2023-06-21T20:07:55.138615Z","shell.execute_reply.started":"2023-06-21T20:07:53.767723Z","shell.execute_reply":"2023-06-21T20:07:55.137727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words('english'))\nnegation_words = [\"didn\", \"weren\", \"doesn't\",\" hasn't\", \"couldn\",\n \"mustn't\", \"isn\", \"hadn't\", \"isn't\", \"wasn't\", \"mightn't\",\n \"couldn't\", \"needn't\", \"haven't\", \"shan't\", \"wouldn't\",\n \"not\", \"no\", \"never\", \"none\", \"nobody\", \"nowhere\", \"nothing\",\"isn'\", \"'t\",\n \"shouldn't\", \"aren't\", \"didn't\", \"didn'\", \"don't\", \"hadn't\", \"won't\", \"wasn\", \"hadn\"]\nstop_words = [ word for word in stop_words if word not in negation_words]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.139966Z","iopub.execute_input":"2023-06-21T20:07:55.140558Z","iopub.status.idle":"2023-06-21T20:07:55.150549Z","shell.execute_reply.started":"2023-06-21T20:07:55.140523Z","shell.execute_reply":"2023-06-21T20:07:55.149576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SentimentData(Dataset):\n    def __init__(self, dataframe_input, tokenizer, max_len, stop_wrods, negation_words, data_type=\"train\"):\n        self.tokenizer = tokenizer\n        self.data_type = data_type\n        self.data = dataframe_input\n        self.text = dataframe_input.review_text.values\n        self.stop_wrods = stop_wrods\n        self.negation_words = negation_words\n\n        if data_type != \"train\":\n            self.review_id = self.data.review_id.values\n\n        if data_type == \"train\":\n            self.targets = self.data.rating\n        self.max_len = max_len\n\n    def __len__(self):\n        return len(self.text)\n    \n    def preprocess_text(self, text):\n        # Remove unnecessary white spaces\n        text = \" \".join(text.split())\n\n        # Remove HTML tags\n        text = re.sub('<[^<]+?>', '', text)\n\n        # Remove URLs\n        text = re.sub(r'http\\S+', '', text)\n\n        # Convert to lowercase\n        text = text.lower()\n\n        # Remove punctuation\n        text = text.translate(str.maketrans('', '', string.punctuation))\n         \n        text = re.sub(r'[^a-zA-Z\\s]', '', text)\n        \n        splited_text = text.split()\n        cleaned_text = []\n        for word in splited_text:\n            if word not in self.stop_wrods:\n                cleaned_text.append(word)\n            elif word in self.negation_words:\n                cleaned_text.append(\"not\")\n        text  = \" \".join(cleaned_text)\n\n\n#         Tokenize the text\n#         tokens = tokenizer.tokenize(text)\n\n#         Remove stopwords\n#         c\n#         tokens = [word for word in tokens if not word in stop_words]\n\n#         Lemmatize the tokens\n#         lemmatizer = WordNetLemmatizer()\n#         tokens = [lemmatizer.lemmatize(word) for word in tokens]\n\n#         Join the tokens back into a string\n#         preprocessed_text = ' '.join(tokens)\n\n        return text\n\n    def __getitem__(self, index):\n        text = str(self.text[index])\n        text = self.preprocess_text(text)\n        \n        inputs = self.tokenizer.encode_plus(\n            text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            pad_to_max_length=True,\n            return_token_type_ids=True\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        token_type_ids = inputs[\"token_type_ids\"]\n        \n        if self.data_type == \"train\":\n             return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'targets':  torch.tensor(self.targets[index], dtype=torch.float)\n\n        }\n\n        else:\n             return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'review_id' : self.review_id[index]\n                }","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.152058Z","iopub.execute_input":"2023-06-21T20:07:55.152565Z","iopub.status.idle":"2023-06-21T20:07:55.168932Z","shell.execute_reply.started":"2023-06-21T20:07:55.152533Z","shell.execute_reply":"2023-06-21T20:07:55.167833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Test Split","metadata":{"execution":{"iopub.status.busy":"2023-03-29T21:36:48.168381Z","iopub.execute_input":"2023-03-29T21:36:48.169703Z","iopub.status.idle":"2023-03-29T21:36:48.177088Z","shell.execute_reply.started":"2023-03-29T21:36:48.169644Z","shell.execute_reply":"2023-03-29T21:36:48.176334Z"}}},{"cell_type":"code","source":"X_train, X_test, _, _ = train_test_split(review_df, review_df, test_size=0.1, random_state=42)\nX_valid, X_test, _, _ = train_test_split(X_test, X_test, test_size=0.5, random_state=42)\n\nX_train.reset_index(inplace=True, drop=True)\nX_test.reset_index(inplace=True, drop=True)\nX_valid.reset_index(inplace=True, drop=True)\n\nprint(\"FULL Dataset: {}\".format(review_df.shape))\nprint(\"TRAIN Dataset: {}\".format(X_train.shape))\nprint(\"Valid Dataset: {}\".format(X_valid.shape))\nprint(\"TEST Dataset: {}\".format(X_test.shape))\n\ntraining_set = SentimentData(X_train, tokenizer, MAX_LEN, stop_words, negation_words )\nvalidation_set = SentimentData(X_valid, tokenizer, MAX_LEN, stop_words, negation_words)\ntesting_set = SentimentData(X_test, tokenizer, MAX_LEN, stop_words, negation_words)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.170673Z","iopub.execute_input":"2023-06-21T20:07:55.171053Z","iopub.status.idle":"2023-06-21T20:07:55.421404Z","shell.execute_reply.started":"2023-06-21T20:07:55.171020Z","shell.execute_reply":"2023-06-21T20:07:55.420413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_params = {'batch_size': TRAIN_BATCH_SIZE,\n                'shuffle': True,\n                'num_workers': 2,\n                'pin_memory':True\n                }\n\ntest_params = {'batch_size': VALID_BATCH_SIZE,\n                'shuffle': True,\n                'num_workers': 2,\n               'pin_memory':True\n                }\n\ntraining_loader = DataLoader(training_set, **train_params)\nvlidation_loader = DataLoader(validation_set, **train_params)\n\ntesting_loader = DataLoader(testing_set, **test_params)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.422863Z","iopub.execute_input":"2023-06-21T20:07:55.423246Z","iopub.status.idle":"2023-06-21T20:07:55.429778Z","shell.execute_reply.started":"2023-06-21T20:07:55.423211Z","shell.execute_reply":"2023-06-21T20:07:55.428576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"class DistilBertClass(torch.nn.Module):\n    def __init__(self):\n        super(DistilBertClass, self).__init__()\n        self.l1 = DistilBertModel.from_pretrained(\"distilbert-base-uncased\")\n        self.pre_classifier = torch.nn.Linear(768, 768)\n        self.dropout = torch.nn.Dropout(0.3)\n        self.classifier = torch.nn.Linear(768, 6)\n\n    def forward(self, input_ids, attention_mask, token_type_ids):\n        output_1 = self.l1(input_ids=input_ids, attention_mask=attention_mask)\n        hidden_state = output_1[0]\n        pooler = hidden_state[:, 0]\n        pooler = self.pre_classifier(pooler)\n        pooler = torch.nn.ReLU()(pooler)\n        pooler = self.dropout(pooler)\n        output = self.classifier(pooler)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.431309Z","iopub.execute_input":"2023-06-21T20:07:55.431774Z","iopub.status.idle":"2023-06-21T20:07:55.441775Z","shell.execute_reply.started":"2023-06-21T20:07:55.431740Z","shell.execute_reply":"2023-06-21T20:07:55.440531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DistilBertClass()\nmodel.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:07:55.442859Z","iopub.execute_input":"2023-06-21T20:07:55.443679Z","iopub.status.idle":"2023-06-21T20:08:02.353974Z","shell.execute_reply.started":"2023-06-21T20:07:55.443644Z","shell.execute_reply":"2023-06-21T20:08:02.352869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = X_train.rating.value_counts()\nclass0 = counts[0]\nclass1 = counts[1]\nclass2 = counts[2]\nclass3 = counts[3]\nclass4 = counts[4]\nclass5 = counts[5]\n\ntotal = class0 + class1 + class1 + class2 + class3 + class4 + class5\n\nweight_for_0 = (1 / class0) * (total / 2.0)\nweight_for_1 = (1 / class1) * (total / 2.0)\nweight_for_2 = (1 / class2) * (total / 2.0)\nweight_for_3 = (1 / class3) * (total / 2.0)\nweight_for_4 = (1 / class4) * (total / 2.0)\nweight_for_5 = (1 / class5) * (total / 2.0)\n\nclass_w = [weight_for_0, weight_for_1, weight_for_2, weight_for_3, weight_for_4, weight_for_5]","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:08:02.355433Z","iopub.execute_input":"2023-06-21T20:08:02.355767Z","iopub.status.idle":"2023-06-21T20:08:02.371098Z","shell.execute_reply.started":"2023-06-21T20:08:02.355733Z","shell.execute_reply":"2023-06-21T20:08:02.369927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the loss function and optimizer\nloss_function = torch.nn.CrossEntropyLoss(weight =  torch.FloatTensor(class_w).to(device))\n# optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)\n\n# optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\noptimizer = torch.optim.AdamW(model.parameters(), lr=3e-5)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:09:14.872754Z","iopub.execute_input":"2023-06-21T20:09:14.873129Z","iopub.status.idle":"2023-06-21T20:09:14.880414Z","shell.execute_reply.started":"2023-06-21T20:09:14.873096Z","shell.execute_reply":"2023-06-21T20:09:14.879500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calcuate_accuracy(preds, targets):\n    n_correct = (preds==targets).sum().item()\n    return n_correct","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:09:16.406338Z","iopub.execute_input":"2023-06-21T20:09:16.406694Z","iopub.status.idle":"2023-06-21T20:09:16.411895Z","shell.execute_reply.started":"2023-06-21T20:09:16.406664Z","shell.execute_reply":"2023-06-21T20:09:16.410838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the training function on the 80% of the dataset for tuning the distilbert model\ndef train(epoch):\n    tr_loss = 0\n    n_correct = 0\n    nb_tr_steps = 0\n    nb_tr_examples = 0\n    model.train()\n    for _,data in tqdm(enumerate(training_loader, 0)):\n        ids = data['ids'].to(device, dtype = torch.long)\n        mask = data['mask'].to(device, dtype = torch.long)\n        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n        targets = data['targets'].to(device, dtype = torch.long)\n\n        outputs = model(ids, mask, token_type_ids)\n        loss = loss_function(outputs, targets)\n        tr_loss += loss.item()\n        big_val, big_idx = torch.max(outputs.data, dim=1)\n        n_correct += calcuate_accuracy(big_idx, targets)\n\n        nb_tr_steps += 1\n        nb_tr_examples+=targets.size(0)\n        \n        if _%5000==0:\n            loss_step = tr_loss/nb_tr_steps\n            accu_step = (n_correct*100)/nb_tr_examples \n            print(f\"Training Loss per 5000 steps: {loss_step}\")\n            print(f\"Training Accuracy per 5000 steps: {accu_step}\")\n\n        optimizer.zero_grad()\n        loss.backward()\n        # # When using GPU\n        optimizer.step()\n\n    print(f'The Total Accuracy for Epoch {epoch}: {(n_correct*100)/nb_tr_examples}')\n    epoch_loss = tr_loss/nb_tr_steps\n    epoch_accu = (n_correct*100)/nb_tr_examples\n    print(f\"Training Loss Epoch: {epoch_loss}\")\n    print(f\"Training Accuracy Epoch: {epoch_accu}\")\n\n    return ","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:09:16.789324Z","iopub.execute_input":"2023-06-21T20:09:16.789659Z","iopub.status.idle":"2023-06-21T20:09:16.800799Z","shell.execute_reply.started":"2023-06-21T20:09:16.789630Z","shell.execute_reply":"2023-06-21T20:09:16.799816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1\nfor epoch in range(EPOCHS):\n    train(epoch)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T20:09:18.557555Z","iopub.execute_input":"2023-06-21T20:09:18.557918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer.save_pretrained(\"./tokinizer3\")\n\n# print('All files saved')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submmetion","metadata":{}},{"cell_type":"code","source":"new_tokenizer = RobertaTokenizer.from_pretrained('/kaggle/working/tokinizer2')\nmodel.load_state_dict(torch.load('/kaggle/working/model_RoBERTa2.pt'))\nmodel.to(\"cpu\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quantized_model = torch.quantization.quantize_dynamic(\n    model, {torch.nn.Linear}, dtype=torch.qint8\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:30:04.206563Z","iopub.execute_input":"2023-06-20T11:30:04.207490Z","iopub.status.idle":"2023-06-20T11:30:05.833193Z","shell.execute_reply.started":"2023-06-20T11:30:04.207449Z","shell.execute_reply":"2023-06-20T11:30:05.831994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quantized_model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv\")\nfinal_test.drop([\"user_id\", \"book_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\", \"n_comments\", \"n_votes\"], axis =1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:17:00.409225Z","iopub.execute_input":"2023-06-20T11:17:00.409850Z","iopub.status.idle":"2023-06-20T11:17:06.424580Z","shell.execute_reply.started":"2023-06-20T11:17:00.409809Z","shell.execute_reply":"2023-06-20T11:17:06.423428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.drop(final_test[1:].index.values, inplace =True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:16:51.227443Z","iopub.execute_input":"2023-06-20T11:16:51.228124Z","iopub.status.idle":"2023-06-20T11:16:51.313662Z","shell.execute_reply.started":"2023-06-20T11:16:51.228079Z","shell.execute_reply":"2023-06-20T11:16:51.312603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.loc[0,\"review_text\"]= \"I don't hate\" ","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:16:51.461670Z","iopub.execute_input":"2023-06-20T11:16:51.462166Z","iopub.status.idle":"2023-06-20T11:16:51.469833Z","shell.execute_reply.started":"2023-06-20T11:16:51.462129Z","shell.execute_reply":"2023-06-20T11:16:51.468613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:17:06.426825Z","iopub.execute_input":"2023-06-20T11:17:06.427263Z","iopub.status.idle":"2023-06-20T11:17:06.440891Z","shell.execute_reply.started":"2023-06-20T11:17:06.427211Z","shell.execute_reply":"2023-06-20T11:17:06.439575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_set = SentimentData(final_test, new_tokenizer, MAX_LEN, stop_words, negation_words, \"test\")\ntest_params = {'batch_size': 32,\n                'shuffle': False,\n                'num_workers': 2,\n               'pin_memory':True\n                }\n\nfinal_test_set = DataLoader(final_test_set, **test_params)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:18:53.175329Z","iopub.execute_input":"2023-06-20T11:18:53.175719Z","iopub.status.idle":"2023-06-20T11:18:53.183159Z","shell.execute_reply.started":"2023-06-20T11:18:53.175684Z","shell.execute_reply":"2023-06-20T11:18:53.181896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:18:53.862165Z","iopub.execute_input":"2023-06-20T11:18:53.863346Z","iopub.status.idle":"2023-06-20T11:18:53.869123Z","shell.execute_reply.started":"2023-06-20T11:18:53.863301Z","shell.execute_reply":"2023-06-20T11:18:53.867895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def final_submmit(model, testing_loader):\n    device = 'cpu'\n    prediction_ids = []\n    preds = []\n    model.eval()\n    \n    with torch.no_grad():\n        for _, data in tqdm(enumerate(testing_loader, 0)):\n            ids = data['ids'].to(device, dtype = torch.long)\n            mask = data['mask'].to(device, dtype = torch.long)\n            token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)\n            review_id = data[\"review_id\"]\n            outputs = model(ids, mask, token_type_ids)\n            softmax_output = F.softmax(outputs, dim=1)\n            predicted_classes = torch.argmax(softmax_output, dim=1)\n            prediction_ids.extend(review_id)\n            preds.extend(predicted_classes.cpu().numpy())\n\n        df = pd.DataFrame({'review_id': prediction_ids , \"rating\": preds })\n        return df\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:30:22.481834Z","iopub.execute_input":"2023-06-20T11:30:22.482830Z","iopub.status.idle":"2023-06-20T11:30:22.493930Z","shell.execute_reply.started":"2023-06-20T11:30:22.482764Z","shell.execute_reply":"2023-06-20T11:30:22.492696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit = final_submmit(quantized_model, final_test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:20:20.239787Z","iopub.status.idle":"2023-06-20T11:20:20.240676Z","shell.execute_reply.started":"2023-06-20T11:20:20.240407Z","shell.execute_reply":"2023-06-20T11:20:20.240435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_submmit.to_csv(\"submmesion2.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T13:22:51.648727Z","iopub.execute_input":"2023-06-03T13:22:51.649401Z","iopub.status.idle":"2023-06-03T13:22:51.656688Z","shell.execute_reply.started":"2023-06-03T13:22:51.649356Z","shell.execute_reply":"2023-06-03T13:22:51.655344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T13:22:20.827493Z","iopub.execute_input":"2023-06-03T13:22:20.828525Z","iopub.status.idle":"2023-06-03T13:22:21.569006Z","shell.execute_reply.started":"2023-06-03T13:22:20.828481Z","shell.execute_reply":"2023-06-03T13:22:21.567868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"0.88 0.4 0.2 0.2 0.2 0.2","metadata":{"execution":{"iopub.status.busy":"2023-03-31T01:09:19.847961Z","iopub.execute_input":"2023-03-31T01:09:19.848501Z","iopub.status.idle":"2023-03-31T01:09:19.870084Z","shell.execute_reply.started":"2023-03-31T01:09:19.848451Z","shell.execute_reply":"2023-03-31T01:09:19.869091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words('english'))\nnegation_words = [\"didn\", \"weren\", \"doesn't\",\" hasn't\", \"couldn\",\n \"mustn't\", \"isn\", \"hadn't\", \"isn't\", \"wasn't\", \"mightn't\",\n \"couldn't\", \"needn't\", \"haven't\", \"shan't\", \"wouldn't\",\n \"not\", \"no\", \"never\", \"none\", \"nobody\", \"nowhere\", \"nothing\",\"isn'\", \"'t\",\n \"shouldn't\", \"aren't\", \"didn't\", \"didn'\", \"don't\", \"hadn't\", \"won't\", \"wasn\", \"hadn\"]\nstop_words = [ word for word in stop_words if word not in negation_words]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Senti_Toknizer():\n    def __init__(self, tokenizer, stop_wrods, negation_words, max_len=256):\n        self.tokenizer = tokenizer\n        self.stop_wrods = stop_wrods\n        self.negation_words = negation_words\n        self.max_len = max_len\n        \n    def preprocess_text(self, text):\n        # Remove unnecessary white spaces\n        text = \" \".join(text.split())\n\n        # Remove HTML tags\n        text = re.sub('<[^<]+?>', '', text)\n\n        # Remove URLs\n        text = re.sub(r'http\\S+', '', text)\n\n        # Convert to lowercase\n        text = text.lower()\n\n        # Remove punctuation\n        text = text.translate(str.maketrans('', '', string.punctuation))\n         \n        text = re.sub(r'[^a-zA-Z\\s]', '', text)\n\n        splited_text = text.split()\n        cleaned_text = []\n        for word in splited_text:\n            if word not in self.stop_wrods:\n                cleaned_text.append(word)\n            elif word in self.negation_words:\n                cleaned_text.append(\"not\")\n        text  = \" \".join(cleaned_text)\n        return text\n\n    def toknize(self, text):\n        text = self.preprocess_text(text)\n        inputs = self.tokenizer.encode_plus(\n            text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            pad_to_max_length=True,\n            return_token_type_ids=True\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        token_type_ids = inputs[\"token_type_ids\"]\n        \n        return {\n        'ids': torch.tensor(ids, dtype=torch.long),\n        'mask': torch.tensor(mask, dtype=torch.long),\n        'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)}\n","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:27:16.841079Z","iopub.execute_input":"2023-06-03T15:27:16.842229Z","iopub.status.idle":"2023-06-03T15:27:16.858732Z","shell.execute_reply.started":"2023-06-03T15:27:16.842184Z","shell.execute_reply":"2023-06-03T15:27:16.857397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = Senti_Toknizer( new_tokenizer, stop_words, negation_words,)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:27:17.377067Z","iopub.execute_input":"2023-06-03T15:27:17.378027Z","iopub.status.idle":"2023-06-03T15:27:17.383047Z","shell.execute_reply.started":"2023-06-03T15:27:17.377987Z","shell.execute_reply":"2023-06-03T15:27:17.381854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = g.toknize(\"this book is not so good but the matrial of the paper is good over all this book is well made\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:47:04.522708Z","iopub.execute_input":"2023-06-03T15:47:04.523412Z","iopub.status.idle":"2023-06-03T15:47:04.529922Z","shell.execute_reply.started":"2023-06-03T15:47:04.523372Z","shell.execute_reply":"2023-06-03T15:47:04.528595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    \n    ids = y['ids'].to(device, dtype = torch.long).unsqueeze(0)\n    mask = y['mask'].to(device, dtype = torch.long).unsqueeze(0)\n    token_type_ids = y['token_type_ids'].to(device, dtype=torch.long).unsqueeze(0)  \n    outputs = model(ids, mask, token_type_ids)\n    \n    softmax_output = F.softmax(outputs, dim=1)\n\n    N_propa = torch.sum(softmax_output[0,0:3]) \n    P_propa = torch.sum(softmax_output[0,3:])\n    \n    print (f\"Positive probability {P_propa}, Negative Propability {N_propa}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:47:04.843702Z","iopub.execute_input":"2023-06-03T15:47:04.844643Z","iopub.status.idle":"2023-06-03T15:47:04.871392Z","shell.execute_reply.started":"2023-06-03T15:47:04.844591Z","shell.execute_reply":"2023-06-03T15:47:04.870002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:30.194256Z","iopub.execute_input":"2023-06-03T15:45:30.195298Z","iopub.status.idle":"2023-06-03T15:45:30.205650Z","shell.execute_reply.started":"2023-06-03T15:45:30.195245Z","shell.execute_reply":"2023-06-03T15:45:30.204215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output[0,0:3]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:43.599451Z","iopub.execute_input":"2023-06-03T15:45:43.599896Z","iopub.status.idle":"2023-06-03T15:45:43.610050Z","shell.execute_reply.started":"2023-06-03T15:45:43.599858Z","shell.execute_reply":"2023-06-03T15:45:43.608691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output[0,3:]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:46:07.410880Z","iopub.execute_input":"2023-06-03T15:46:07.411274Z","iopub.status.idle":"2023-06-03T15:46:07.423226Z","shell.execute_reply.started":"2023-06-03T15:46:07.411237Z","shell.execute_reply":"2023-06-03T15:46:07.421734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax_output","metadata":{"execution":{"iopub.status.busy":"2023-06-03T15:45:59.406351Z","iopub.execute_input":"2023-06-03T15:45:59.407354Z","iopub.status.idle":"2023-06-03T15:45:59.417059Z","shell.execute_reply.started":"2023-06-03T15:45:59.407313Z","shell.execute_reply":"2023-06-03T15:45:59.415810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(quantized_model, 'qun_model_RoBERTa3w.pth')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:55.126088Z","iopub.execute_input":"2023-06-20T11:58:55.127133Z","iopub.status.idle":"2023-06-20T11:58:56.139478Z","shell.execute_reply.started":"2023-06-20T11:58:55.127079Z","shell.execute_reply":"2023-06-20T11:58:56.138134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = torch.load('/kaggle/working/qun_model_RoBERTa3w.pth')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:57.926845Z","iopub.execute_input":"2023-06-20T11:58:57.927882Z","iopub.status.idle":"2023-06-20T11:58:58.690236Z","shell.execute_reply.started":"2023-06-20T11:58:57.927827Z","shell.execute_reply":"2023-06-20T11:58:58.689094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c","metadata":{"execution":{"iopub.status.busy":"2023-06-20T11:58:59.725125Z","iopub.execute_input":"2023-06-20T11:58:59.726378Z","iopub.status.idle":"2023-06-20T11:59:00.178388Z","shell.execute_reply.started":"2023-06-20T11:58:59.726332Z","shell.execute_reply":"2023-06-20T11:59:00.177070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}