{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Goodreads Books Reviews\nGoodreads Books Review Rating Prediction","metadata":{}},{"cell_type":"code","source":"!pip install -q transformers --quiet","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn \nimport numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport pandas as pd \nfrom tqdm import tqdm \nfrom transformers import BertModel ,BertTokenizer,AdamW, get_linear_schedule_with_warmup, set_seed\nfrom pylab import rcParams\nimport seaborn as sns \nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nimport random \nimport numpy as np","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_all(seed_value):\n    random.seed(seed_value) \n    np.random.seed(seed_value) \n    torch.manual_seed(seed_value)\n    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True \n        torch.backends.cudnn.benchmark = False\n\nseed =42\nseed_all(seed)","metadata":{"id":"fd5HV7WrwsWA","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    NB_EPOCHS = 6\n    LR = 3e-5\n    EPS=1e-8\n    MAX_LEN = 110\n    N_SPLITS = 4\n    TRAIN_BS = 60\n    VALID_BS = 40\n    MODEL_NAME = 'bert-base-cased'\n    TRAIN_FILE = '/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv'\n    TEST_FILE = '/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv'\n    SUB_FILE = '/kaggle/input/goodreads-books-reviews-290312/goodreads_sample_submission.csv'\n    TOKENIZER = BertTokenizer.from_pretrained(MODEL_NAME)","metadata":{"id":"bGSXOKZ-wsTK","outputId":"08bba0d0-eef5-4bce-b21a-db0d5ac96e66","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"id":"RrXJK5rYwsQu","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams[\"figure.figsize\"] = 12,8","metadata":{"id":"idTjDkxHwsOZ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df =  pd.read_csv(Config.TRAIN_FILE)\ndf = df.drop(['user_id','book_id','date_added','date_updated','read_at', \\\n              'started_at','n_votes','n_comments'], axis =1)\ndf = df[['review_id','review_text','rating']]\ndf = df.sample(frac=0.0167)  # Use only approximately 15,000 sample data to train for now\ndf.head()","metadata":{"id":"ng9q3dtrwsL9","outputId":"ea9917e4-e5f0-4f8a-df21-2d3a71e31e3b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.rating.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(df.rating)","metadata":{"id":"fO8NKehVwsJU","outputId":"71da441b-2bbd-45cb-b9a6-e5a7b9b287cc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_ord = [0,1,2,3,4,5]\ndf['rating'] = df.rating.astype('category').cat.codes\n\nY = to_categorical(df['rating'])\n\nfor i in range(len(labels_ord)) :     \n     df[labels_ord[i]] = Y[:,i]","metadata":{"id":"u6kap01ywsG3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(3)","metadata":{"id":"ByPWslNpwsEu","outputId":"5ed62513-b33b-4410-b940-3ffc625d0c52","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can count the max lenght size:**\n","metadata":{"id":"6YATne9Oxigx"}},{"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained(Config.MODEL_NAME,use_fast=True)","metadata":{"id":"53-wmMfswsCo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"token_lens = []\n\nfor txt in df.review_text:\n    tokens = tokenizer.encode(txt, max_length=512)\n    token_lens.append(len(tokens)) ","metadata":{"id":"BnXh0qT4wsAI","outputId":"ab31c0ae-ede7-4130-fcf5-2c9a5aecc400","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(token_lens) ","metadata":{"id":"VLD8vxa4wr-F","outputId":"01d268ac-0619-4ef1-e39b-2dcdd5c91946","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GRDataset(Dataset):\n    def __init__(self, review_text, targets=None, is_test=False):\n        self.review_text = review_text\n        self.targets = targets\n        self.is_test = is_test\n        self.tokenizer = Config.TOKENIZER\n        self.max_len = Config.MAX_LEN\n    \n    def __len__(self):\n        return len(self.review_text)\n    \n    def __getitem__(self, idx):\n        review_text = str(self.review_text[idx])\n        review_text = ' '.join(review_text.split())\n        global inputs\n       \n        inputs = self.tokenizer(\n                            review_text,\n                            add_special_tokens=True,\n                            max_length=self.max_len,\n                            padding=\"max_length\" ,\n                            truncation = True ,\n                            pad_to_max_length=True, \n                            )\n        \n        ids = torch.tensor(inputs['input_ids'], dtype=torch.long)\n        mask = torch.tensor(inputs['attention_mask'], dtype=torch.long)\n        token_type = torch.tensor(inputs['token_type_ids'], dtype=torch.long)\n     \n        \n   \n        if self.is_test:\n            return {\n                'ids': ids,\n                'mask': mask,\n                'token_type':token_type\n               \n            }\n        else:    \n            targets = torch.tensor(self.targets[idx], dtype=torch.long)\n            return {\n                'ids': ids,\n                'mask': mask,\n                'token_type':token_type,\n                'targets': targets\n            }","metadata":{"id":"n79H7uZiwr8Q","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GRDataloader(df  , batch_size , is_test=False):\n    dataset = GRDataset(df[\"review_text\"].values , df[\"rating\"].values  , is_test)\n    dataloader = DataLoader(dataset , batch_size , shuffle=False)\n    return dataloader","metadata":{"id":"XrzPrq95wr6m","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloader = GRDataloader(df , 1)","metadata":{"id":"KNZILmtawr4l","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = next(iter(dataloader))\ndata","metadata":{"id":"Ljq4TEfAwr2N","outputId":"c356c1d0-0713-4805-cea4-ce34ff940800","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\nclass GBVClassifier(nn.Module):\n    def __init__(self , n_classes):\n        super(GBVClassifier,self).__init__()\n        self.bert = BertModel.from_pretrained(Config.MODEL_NAME , return_dict=False)\n        self.drop1=  nn.Dropout(p=0.7)\n        self.drop2=  nn.Dropout(p=0.8)\n        self.out = nn.Linear(768,n_classes)\n        \n\n    def forward(self , input_ids, attention_mask ,token_type_ids):\n        \n        _,sortie = self.bert(input_ids,attention_mask,token_type_ids)\n\n        output1 = self.drop1(sortie)\n        output1 = self.out(output1)\n        \n        output2 = self.drop2(sortie)\n        output2 = self.out(output2)\n        \n        output= output1.add(output2)/2\n     \n        \n        return output\n        \n\"\"\" ","metadata":{"id":"v1qeQYehwr0q","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GRClassifier(nn.Module):\n    def __init__(self , n_classes):\n        super(GRClassifier,self).__init__()\n        self.bert = BertModel.from_pretrained(Config.MODEL_NAME , return_dict=False)\n        self.drop=  nn.Dropout(p=0.8)\n        self.out = nn.Linear(768,n_classes)\n        \n\n    def forward(self , input_ids, attention_mask ,token_type_ids):\n        \n        _,pooledout = self.bert(input_ids,attention_mask,token_type_ids)\n\n        output = self.drop(pooledout)\n        output = self.out(output)\n        \n        return output","metadata":{"id":"cqJf4oWtwrw7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = GRClassifier(len(labels_ord))\nmodel.to(device)","metadata":{"id":"DZDawh_ywrtR","outputId":"4c6d681e-2bea-4fd3-ecbc-6d9dffdae2ef","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(outputs, labels):\n     \n    return nn.CrossEntropyLoss()(outputs, labels)\n\ndef yield_optimizer(model):\n    \"\"\"\n    Returns optimizer for specific parameters\n    \"\"\"\n    param_optimizer = list(model.named_parameters())\n    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n\n    optimizer_parameters = [\n        {\n            \"params\": [\n                p for n, p in param_optimizer if not any(nd in n for nd in no_decay)\n            ],\n            \"weight_decay\": 0.001,\n        },\n        {\n            \"params\": [\n                p for n, p in param_optimizer if any(nd in n for nd in no_decay)\n            ],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    return AdamW(optimizer_parameters, lr=Config.LR, eps=Config.EPS) ","metadata":{"id":"InGkZqMRwrqZ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch (model , data_loader , loss_fn , optimizer , device , scheduler , n_examples):\n    model.train()\n    losses = []\n    correct_predictions = 0\n\n    for step , d in tqdm(enumerate(data_loader) , total=len(data_loader)):\n        \n        input_ids =d['ids'].to(device) \n        token_type_ids = d['token_type'].to(device)\n        attention_mask = d['mask'].to(device)\n        targets = d['targets'].to(device)\n\n        outputs = model(\n             input_ids ,\n            attention_mask ,\n            token_type_ids)\n        \n        _, pred =  torch.max(outputs , dim=1)\n\n        loss = loss_fn(outputs , targets)\n        correct_predictions +=torch.sum(pred ==targets)\n        losses.append(loss.item())\n\n\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        optimizer.zero_grad()\n        \n    return correct_predictions.double() / n_examples  , np.mean(losses)","metadata":{"id":"4UA2LtiWxuaT","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_model (model , data_loader , loss_fn , device , n_examples):\n    \n    model.eval()\n  \n    losses = []\n    correct_predictions = 0\n\n    with torch.no_grad() :\n        \n        for step , d in tqdm(enumerate(data_loader) , total=len(data_loader)):\n            \n            input_ids =d['ids'].to(device)\n            token_type_ids = d['token_type'].to(device)\n            attention_mask = d['mask'].to(device)\n            targets = d['targets'].to(device)\n        \n            outputs = model(\n                   input_ids ,\n                attention_mask ,\n                token_type_ids  )\n\n            _, pred =  torch.max(outputs , dim=1)\n            loss = loss_fn(outputs , targets)\n            correct_predictions +=torch.sum(pred ==targets)\n            losses.append(loss.item())\n\n    return correct_predictions.double() / n_examples  , np.mean(losses)","metadata":{"id":"hKYFYFXaxuVT","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model , df , epochs):\n    \n    \n    best_accuracy = 0\n    some_val =0\n    \n    for epoch in range(epochs):\n\n        print(f'Epoch {epoch + 1}')\n\n        kf = StratifiedKFold(n_splits=4,random_state=seed, shuffle=True)  # random_state=seed, ,shuffle=False\n\n        for step , (train, valid ) in enumerate(kf.split(df , df[\"rating\"])) :\n          \n            train_data_loader= GRDataloader(df.iloc[train],Config.TRAIN_BS)\n            validation_data_loader = GRDataloader(df.iloc[valid],Config.VALID_BS)\n\n            nb_train_steps = int(len(train_data_loader) /Config.TRAIN_BS * epochs)\n            optimizer = yield_optimizer(model)\n            scheduler = get_linear_schedule_with_warmup(\n                                        optimizer,\n                                        num_warmup_steps=0,\n                                        num_training_steps=nb_train_steps)\n\n            train_acc,train_loss = train_epoch(model,train_data_loader,loss_fn,optimizer,device,scheduler,len(df.iloc[train])) \n            \n            print(f\"Train accuracy {train_acc} ,Train Loss {train_loss}\")\n                \n            val_acc, val_loss = eval_model(model,validation_data_loader,loss_fn,device,len(df.iloc[valid]))\n            \n            print(f\"Validation accuracy {val_acc} , Validation loss {val_loss}\")\n\n          \n            if  val_acc > best_accuracy:\n\n                torch.save(model.state_dict(),'best_model.bin')\n                best_accuracy = val_acc\n                print(f\"Best accuracy {best_accuracy}\")","metadata":{"id":"gFE19LwGxuQc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(model, df, Config.NB_EPOCHS)","metadata":{"id":"bx830puAySXt","outputId":"0c621d8d-ef93-45ec-e84e-c832c0a214ce","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## For Real time use Config.NB_EPOCHS instead 1\n#train(model , df , 1)","metadata":{"id":"oofFHuYWxuNw","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(Config.TEST_FILE)\ntest = test.drop(['user_id','book_id','date_added','date_updated','read_at', \\\n                  'started_at','n_votes','n_comments'], axis =1)\ntest = test[['review_id','review_text']]\ntest.head()","metadata":{"id":"8dO6UVP5yY0H","outputId":"8005e8b7-481a-4697-d714-adc240b92abe","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test[['review_text']]\ntest.head()","metadata":{"id":"nrDernFtxuLd","outputId":"3941b68d-f8f3-4094-c275-cfb86e43c2ed","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = GRClassifier(len(labels_ord))\nmodel.load_state_dict(torch.load('./best_model.bin'))\nmodel = model.to(device)","metadata":{"id":"qoeftcnVxuIW","outputId":"0b28bd6b-6b7f-461b-e101-b2f886c228d5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ReviewTestDataloader(df, batch_size , is_test=True):\n    dataset = GRDataset(df[\"review_text\"].values ,None  , is_test)\n    dataloader = DataLoader(dataset , batch_size , shuffle=False)\n    return dataloader","metadata":{"id":"3cIIw5FRyfuW","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdataloader = ReviewTestDataloader(test, 1)","metadata":{"id":"FGzWMpSKxuD9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba= []","metadata":{"id":"5-MNBcz0xuBS","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(model, df_test):\n    \n    model = model.eval()\n  \n    predictions = []\n  \n    data_loader = ReviewTestDataloader(df_test, 1)\n   \n    with torch.no_grad():\n        for d in data_loader:\n            \n            input_ids = d[\"ids\"].to(device)\n            attention_mask = d[\"mask\"].to(device)\n            token_type_ids = d[\"token_type\"].to(device)\n            \n            outputs = model(\n                            input_ids,\n                            attention_mask ,\n                            token_type_ids\n                              )\n            proba.append(outputs)\n            _, preds = torch.max(outputs, dim=1)","metadata":{"id":"n_uflUF_ymQW","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nget_predictions(model,test)","metadata":{"id":"ys1p3WI5yqh4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tab=[]","metadata":{"id":"J4jUQngeyrZY","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in proba:\n    tab.append(i.flatten().tolist())","metadata":{"id":"4XXWsbbJxt-O","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_pd = pd.DataFrame(data=tab , columns=labels_ord)","metadata":{"id":"OytOj0gXyKiz","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_pd.head()","metadata":{"id":"6pYYCJk0y3U3","outputId":"ea79d854-f076-474d-d78c-0b3d192b788b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict_pd[3] = predict_pd[3]*1.65\n# predict_pd[4] = predict_pd[4]*2.75\n# predict_pd[5] = predict_pd[5]*3.35","metadata":{"id":"Sjcn9eA4y87f","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_prediction = predict_pd.idxmax(axis=1)","metadata":{"id":"VZ92LNpqyKgR","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(Config.SUB_FILE)\nsub[\"rating\"]= get_prediction\nsub.to_csv(\"submission_new1.csv\",index=False)\nsub.head()","metadata":{"id":"f-xZPPtmzF-A","outputId":"88af8f9e-c892-4bb9-e860-3660c1218607","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"rating\"].value_counts()","metadata":{"id":"jyFvW_WdyKZK","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"d6AOgbk9yKWY"},"execution_count":null,"outputs":[]}]}