{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Please Upvote if u feel this is useful !**","metadata":{}},{"cell_type":"markdown","source":"# Set Env","metadata":{}},{"cell_type":"code","source":"!pip install transformers torchsummary wordcloud tweet-preprocessor","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:39:52.877059Z","iopub.execute_input":"2023-04-19T09:39:52.877895Z","iopub.status.idle":"2023-04-19T09:40:02.674058Z","shell.execute_reply.started":"2023-04-19T09:39:52.877865Z","shell.execute_reply":"2023-04-19T09:40:02.672811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, random_split\nimport transformers\nfrom transformers import BertModel, BertTokenizerFast\nfrom transformers import DistilBertModel, DistilBertTokenizerFast\n\n\nimport nltk\nfrom nltk import word_tokenize, FreqDist\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import TweetTokenizer\nimport preprocessor as p # tweet preprocessor\nfrom wordcloud import WordCloud\n\nimport zipfile\nimport glob\nimport cv2\nimport gc\nimport random\nimport math\nfrom tqdm.notebook import tqdm\nimport json\nimport re\nimport os\nfrom collections import Counter\nimport statistics\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:40:28.450413Z","iopub.execute_input":"2023-04-19T09:40:28.451390Z","iopub.status.idle":"2023-04-19T09:40:28.477656Z","shell.execute_reply.started":"2023-04-19T09:40:28.451334Z","shell.execute_reply":"2023-04-19T09:40:28.476485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download\nnltk.download('wordnet')\nnltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:30:19.901049Z","iopub.execute_input":"2023-04-19T09:30:19.902689Z","iopub.status.idle":"2023-04-19T09:30:20.001526Z","shell.execute_reply.started":"2023-04-19T09:30:19.902645Z","shell.execute_reply":"2023-04-19T09:30:20.000398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 33\nEPOCHS = 4\nLEARNING_RATE = 1e-5\nEARLY_STOP_LIMIT = 20\nVALIDATION_RATIO = 0.03\nMODEL_PATH = \"./best_model.pt\"\n\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.manual_seed(1004)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(104)\n\nprint(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:51:10.016659Z","iopub.execute_input":"2023-04-19T10:51:10.017127Z","iopub.status.idle":"2023-04-19T10:51:10.024924Z","shell.execute_reply.started":"2023-04-19T10:51:10.017085Z","shell.execute_reply":"2023-04-19T10:51:10.023645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords_list = stopwords.words('english')\nadditional_stopwords_list = []\nstopwords_list.extend(additional_stopwords_list)\nprint(stopwords_list)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:30:20.076300Z","iopub.execute_input":"2023-04-19T09:30:20.077345Z","iopub.status.idle":"2023-04-19T09:30:20.086153Z","shell.execute_reply.started":"2023-04-19T09:30:20.077304Z","shell.execute_reply":"2023-04-19T09:30:20.084921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/goodreads-books-reviews-290312/goodreads_train.csv')\ntest_df = pd.read_csv('../input/goodreads-books-reviews-290312/goodreads_test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:30:20.087851Z","iopub.execute_input":"2023-04-19T09:30:20.088532Z","iopub.status.idle":"2023-04-19T09:31:00.721053Z","shell.execute_reply.started":"2023-04-19T09:30:20.088496Z","shell.execute_reply":"2023-04-19T09:31:00.720001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_df))\nprint(len(test_df))\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:31:00.722655Z","iopub.execute_input":"2023-04-19T09:31:00.723731Z","iopub.status.idle":"2023-04-19T09:31:00.753464Z","shell.execute_reply.started":"2023-04-19T09:31:00.723687Z","shell.execute_reply":"2023-04-19T09:31:00.752345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"len_of_sentence = [len(row[\"review_text\"]) for row in train_df.iloc]\nprint(min(len_of_sentence), max(len_of_sentence))","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:31:00.754925Z","iopub.execute_input":"2023-04-19T09:31:00.755375Z","iopub.status.idle":"2023-04-19T09:32:46.328397Z","shell.execute_reply.started":"2023-04-19T09:31:00.755337Z","shell.execute_reply":"2023-04-19T09:32:46.327108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(\"Median of sentence length : {}\".format(statistics.median(len_of_sentence)))\nprint(\"Mean of sentence length : {:.2f}\".format(statistics.mean(len_of_sentence)))\npd.Series(len_of_sentence).hist(bins=20000)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:32:46.329778Z","iopub.execute_input":"2023-04-19T09:32:46.330253Z","iopub.status.idle":"2023-04-19T09:33:21.030132Z","shell.execute_reply.started":"2023-04-19T09:32:46.330213Z","shell.execute_reply":"2023-04-19T09:33:21.029180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### WordCloud","metadata":{}},{"cell_type":"code","source":"sentence_list = train_df[\"review_text\"].tolist()\nall_words_counter = Counter()\nfor idx, sentence in enumerate(tqdm(sentence_list)):\n    sentence = sentence.lower()\n    sentence = sentence.replace(\"\\n\", \"\")\n    words_list = sentence.split(\" \")\n    words_list = [word for word in words_list if word not in stopwords_list]\n    words_counter = Counter(words_list)\n    all_words_counter.update(words_counter)\n\nwc_object = WordCloud(width=1200, height=900)\nwc_object.generate_from_frequencies(all_words_counter)\n\n\nplt.figure(figsize=(60,40))\nplt.imshow(wc_object)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:33:21.035767Z","iopub.execute_input":"2023-04-19T09:33:21.038001Z","iopub.status.idle":"2023-04-19T09:39:52.876071Z","shell.execute_reply.started":"2023-04-19T09:33:21.037961Z","shell.execute_reply":"2023-04-19T09:39:52.875170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"rating\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:00.974560Z","iopub.execute_input":"2023-04-19T09:41:00.975795Z","iopub.status.idle":"2023-04-19T09:41:00.998143Z","shell.execute_reply.started":"2023-04-19T09:41:00.975726Z","shell.execute_reply":"2023-04-19T09:41:00.997107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"votes_count_list = train_df[\"n_votes\"].value_counts().tolist()\n\nprint(\"Median of votes : {}\".format(statistics.median(votes_count_list)))\nprint(\"Mean of votes : {:.1f}\".format(statistics.mean(votes_count_list)))\npd.Series(votes_count_list).plot.kde(ind=votes_count_list, title=\"Distribution of votes\")","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:01.894389Z","iopub.execute_input":"2023-04-19T09:41:01.895508Z","iopub.status.idle":"2023-04-19T09:41:02.178005Z","shell.execute_reply.started":"2023-04-19T09:41:01.895457Z","shell.execute_reply":"2023-04-19T09:41:02.176977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comments_count_list = train_df[\"n_comments\"].value_counts().tolist()\n\nprint(\"Median of votes : {}\".format(statistics.median(comments_count_list)))\nprint(\"Mean of votes : {:.1f}\".format(statistics.mean(comments_count_list)))\npd.Series(comments_count_list).plot.kde(ind=comments_count_list, title=\"Distribution of comments numbers\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:03.540315Z","iopub.execute_input":"2023-04-19T09:41:03.542241Z","iopub.status.idle":"2023-04-19T09:41:03.778916Z","shell.execute_reply.started":"2023-04-19T09:41:03.542192Z","shell.execute_reply":"2023-04-19T09:41:03.777999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"markdown","source":"### - Step 2 do\n1. lower all setence\n2. remove stopwords\n3. remove url but tokenize\n4. only alphanumeric (line-feed is removed in this process)\n5. replace all multiple spaces to single space\n--*--","metadata":{}},{"cell_type":"code","source":"def remove_stopwords(text, stopwords_list=stopwords_list):\n    \n    text_list = text.split(\" \")\n    res_list = [word for word in text_list if word not in stopwords_list]\n    res_text = ' '.join(res_list)\n\n    return res_text","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:10.893971Z","iopub.execute_input":"2023-04-19T09:41:10.894662Z","iopub.status.idle":"2023-04-19T09:41:10.902966Z","shell.execute_reply.started":"2023-04-19T09:41:10.894624Z","shell.execute_reply":"2023-04-19T09:41:10.901966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_only_in_regex(text, regex):\n    p = re.compile(regex)\n    m = p.findall(text)\n    if m is None:\n        return \"\"\n    \n    else:\n        processed_text = \" \".join(m)\n        return processed_text","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:11.179934Z","iopub.execute_input":"2023-04-19T09:41:11.180312Z","iopub.status.idle":"2023-04-19T09:41:11.186057Z","shell.execute_reply.started":"2023-04-19T09:41:11.180279Z","shell.execute_reply":"2023-04-19T09:41:11.184735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_text(target_df: pd.DataFrame) -> pd.DataFrame:\n    # at first, lower all sentence\n\n    # remove stopword\n    # remove hashtag but leave keyword\n    # tokenize url, mentions, emoji.. etc \n    \n    # only alphanumeric and $TOKEN$\n    # remove multiple spaces\n    \n    alphanumeric_token_regex = \"[a-zA-Z0-9]+|\\$[A-Z]+\\$\"\n    p.set_options(p.OPT.URL, p.OPT.EMOJI)\n\n\n    for idx in tqdm(range(0, len(target_df))):\n        raw_text = target_df.iloc[idx][\"review_text\"]\n        fixed_text = raw_text.lower()\n        \n        fixed_text = remove_stopwords(fixed_text)\n        \n        fixed_text = p.tokenize(fixed_text)\n\n        fixed_text = get_only_in_regex(fixed_text, alphanumeric_token_regex)\n        fixed_text = re.sub(\" +\", \" \", fixed_text)\n\n        target_df.at[idx, 'review_text'] = fixed_text\n\n    return target_df","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:11.940465Z","iopub.execute_input":"2023-04-19T09:41:11.942276Z","iopub.status.idle":"2023-04-19T09:41:11.950104Z","shell.execute_reply.started":"2023-04-19T09:41:11.942210Z","shell.execute_reply":"2023-04-19T09:41:11.949029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_train_df = preprocess_text(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T09:41:21.188403Z","iopub.execute_input":"2023-04-19T09:41:21.189361Z","iopub.status.idle":"2023-04-19T09:57:08.249038Z","shell.execute_reply.started":"2023-04-19T09:41:21.189307Z","shell.execute_reply":"2023-04-19T09:57:08.246825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_train_df[\"review_text\"].isnull().values.any()","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:26:04.853990Z","iopub.execute_input":"2023-04-19T10:26:04.855041Z","iopub.status.idle":"2023-04-19T10:26:04.976198Z","shell.execute_reply.started":"2023-04-19T10:26:04.854992Z","shell.execute_reply":"2023-04-19T10:26:04.974782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_of_word = [len(row[\"review_text\"].split(\" \")) for row in fixed_train_df.iloc]\nprint(min(num_of_word), max(num_of_word))\nprint(\"===========\")\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:26:05.340427Z","iopub.execute_input":"2023-04-19T10:26:05.341343Z","iopub.status.idle":"2023-04-19T10:27:51.463127Z","shell.execute_reply.started":"2023-04-19T10:26:05.341282Z","shell.execute_reply":"2023-04-19T10:27:51.462017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top10_idx = 9*int(len(num_of_word)/10)\nsorted(num_of_word)[top10_idx]","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:27:51.465237Z","iopub.execute_input":"2023-04-19T10:27:51.465635Z","iopub.status.idle":"2023-04-19T10:27:51.617428Z","shell.execute_reply.started":"2023-04-19T10:27:51.465596Z","shell.execute_reply":"2023-04-19T10:27:51.615884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Median of token quantity : {}\".format(statistics.median(num_of_word)))\nprint(\"Mean of token quantity : {:.2f}\".format(statistics.mean(num_of_word)))\npd.Series(num_of_word).hist(bins=1870)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:27:51.618987Z","iopub.execute_input":"2023-04-19T10:27:51.620011Z","iopub.status.idle":"2023-04-19T10:27:55.801469Z","shell.execute_reply.started":"2023-04-19T10:27:51.619895Z","shell.execute_reply":"2023-04-19T10:27:55.799997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Structurize Dataset","metadata":{}},{"cell_type":"code","source":"model_name = 'distilbert-base-cased'\nbert_tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:21.926315Z","iopub.execute_input":"2023-04-19T10:49:21.927275Z","iopub.status.idle":"2023-04-19T10:49:23.408931Z","shell.execute_reply.started":"2023-04-19T10:49:21.927216Z","shell.execute_reply":"2023-04-19T10:49:23.407875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ReviewDataset(Dataset):\n    def __init__(self, df, is_train):\n        self.raw_df = df\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.raw_df)\n\n    def __getitem__(self, idx):\n\n        row_data = self.raw_df.iloc[idx]\n        text = row_data[\"review_text\"]\n        n_votes = row_data[\"n_votes\"]\n        n_comments = row_data[\"n_comments\"]\n        \n        if self.is_train == False:\n            return text, n_votes, n_comments\n        else:\n            rating = row_data[\"rating\"]\n            return text, n_votes, n_comments, rating\n\n\n\ndef collate_fn(batch):\n    batch = np.array(batch)\n\n    batch_text_list = batch[:, 0]\n    batch_popularity_list = batch[:, 1:3]  # [votes, comments]\n\n    text_collated = bert_tokenizer(batch_text_list.tolist(), return_tensors='pt', max_length=192, padding='max_length', truncation=True)\n    popularity_collated = torch.LongTensor(batch_popularity_list.astype(np.int32))\n    \n    batch_label_list = batch[:, 3]\n    label_collated = torch.LongTensor(batch_label_list.astype(np.int32))\n    return text_collated, popularity_collated, label_collated\n","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:26.192268Z","iopub.execute_input":"2023-04-19T10:49:26.193215Z","iopub.status.idle":"2023-04-19T10:49:26.203441Z","shell.execute_reply.started":"2023-04-19T10:49:26.193163Z","shell.execute_reply":"2023-04-19T10:49:26.201998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = ReviewDataset(fixed_train_df, is_train=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:26.894393Z","iopub.execute_input":"2023-04-19T10:49:26.895025Z","iopub.status.idle":"2023-04-19T10:49:26.901828Z","shell.execute_reply.started":"2023-04-19T10:49:26.894983Z","shell.execute_reply":"2023-04-19T10:49:26.900646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_whole_train_dataset = len(train_dataset)\nlen_validation_dataset = int(VALIDATION_RATIO*len_whole_train_dataset)\nlen_train_dataset = len(train_dataset) - int(VALIDATION_RATIO*len_whole_train_dataset)\ntrain_dataset, val_dataset = random_split(train_dataset, [len_train_dataset, len_validation_dataset])","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:28.342759Z","iopub.execute_input":"2023-04-19T10:49:28.343925Z","iopub.status.idle":"2023-04-19T10:49:28.441458Z","shell.execute_reply.started":"2023-04-19T10:49:28.343873Z","shell.execute_reply":"2023-04-19T10:49:28.440401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(\n    dataset=train_dataset, \n    batch_size=BATCH_SIZE, \n    collate_fn=collate_fn,\n    shuffle=True,\n    drop_last=True,\n    pin_memory=True,\n    )\n\nval_dataloader = DataLoader(\n    dataset=val_dataset,\n    batch_size=BATCH_SIZE,\n    collate_fn=collate_fn,\n    shuffle=True,\n    drop_last=True,\n    pin_memory=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:28.942552Z","iopub.execute_input":"2023-04-19T10:49:28.943343Z","iopub.status.idle":"2023-04-19T10:49:28.949408Z","shell.execute_reply.started":"2023-04-19T10:49:28.943303Z","shell.execute_reply":"2023-04-19T10:49:28.948309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for text_seq, popularity_info, labels in train_dataloader:\n    print(text_seq)\n    print(\"============\")\n    print(popularity_info)\n    print(\"============\")\n    print(labels)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:49:30.004518Z","iopub.execute_input":"2023-04-19T10:49:30.005137Z","iopub.status.idle":"2023-04-19T10:49:32.817701Z","shell.execute_reply.started":"2023-04-19T10:49:30.005098Z","shell.execute_reply":"2023-04-19T10:49:32.816748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"class LangClassifyModel(nn.Module):\n    def __init__(self, num_classes, freeze_bert=True):\n        super(LangClassifyModel, self).__init__()\n        \n        self.bert_model = DistilBertModel.from_pretrained(model_name)\n        self.bi_lstm = nn.LSTM(768, 256, batch_first=True, num_layers=3, bidirectional=False, dropout=0.5)\n        self.fc_1 = nn.Linear(256, 120)\n        self.fc_2 = nn.Linear(2,8)\n        self.fc_final = nn.Linear(128, num_classes)\n        self.dropout = nn.Dropout(0.5)\n        \n        if freeze_bert:\n            for param in self.bert_model.parameters():\n                param.requires_grad = False\n        else:\n            for param in self.bert_model.parameters():\n                param.requires_grad = True\n                \n    def forward(self, text_seq, popularity_info):\n        bert_output = self.bert_model(**text_seq)\n        x1 = bert_output['last_hidden_state']\n        x1, _ = self.bi_lstm(x1)\n        x1 = x1[:, 0]\n        x1 = self.fc_1(x1)\n        x1 = F.relu(x1)\n        x1 = self.dropout(x1)\n    \n        popularity_info = popularity_info.to(torch.float32)\n        x2 = self.fc_2(popularity_info)\n        x2 = F.relu(x2)\n        \n        x = torch.cat([x1, x2], 1)\n        x = self.dropout(x)\n        x = self.fc_final(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:50:02.401773Z","iopub.execute_input":"2023-04-19T10:50:02.402775Z","iopub.status.idle":"2023-04-19T10:50:02.412823Z","shell.execute_reply.started":"2023-04-19T10:50:02.402738Z","shell.execute_reply":"2023-04-19T10:50:02.411771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LangClassifyModel(num_classes=6, freeze_bert=False)\nmodel = model.to(DEVICE)\n\n# if u continuing stopped trainting, just load saved model\n# model.load_state_dict(torch.load(MODEL_PATH)) ","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:50:03.117837Z","iopub.execute_input":"2023-04-19T10:50:03.118816Z","iopub.status.idle":"2023-04-19T10:50:06.355656Z","shell.execute_reply.started":"2023-04-19T10:50:03.118777Z","shell.execute_reply":"2023-04-19T10:50:06.354602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(model, train_dataloader, val_dataloader):\n    \n    optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n    error_func = nn.CrossEntropyLoss().to(DEVICE)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3)\n\n\n    modeling_progress = {\n    'train_acc': [],\n    'val_acc': [],\n    'val_loss': [],\n    }\n\n    best_val_acc = 0\n    early_stop_counter = 0\n\n    for epoch in range(EPOCHS):\n        train_acc = 0\n        val_acc = 0\n        \n        \n        train_correct = 0\n        val_correct = 0\n\n        val_loss = 0\n\n\n        # for train\n        model.train()\n        for batch_idx, (text_seqs, popularity_info, labels) in enumerate(tqdm(train_dataloader)):\n            \n            input_ids = text_seqs['input_ids'].to(DEVICE)\n            attention_mask = text_seqs['attention_mask'].to(DEVICE)\n            labels = labels.to(DEVICE)\n            \n            inputs = {\"input_ids\":input_ids, \"attention_mask\":attention_mask}\n            popularity_inputs = popularity_info.to(DEVICE)\n            \n            optimizer.zero_grad()\n            \n            outputs = model(inputs, popularity_inputs)\n            \n            loss = error_func(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            # Total correct predictions\n            predicted = torch.sigmoid(outputs).argmax(dim=1)\n            train_correct += (predicted == labels).sum()\n            \n            if batch_idx % 1000 == 0:\n                print('Epoch : {} [{}/{}]\\t Accuracy:{:.3f}%'.format(\n                    epoch, batch_idx*BATCH_SIZE, len(train_dataloader.dataset), float(train_correct*100) / float(BATCH_SIZE*(batch_idx+1))\n                        )\n                     )\n\n                    \n        # for validation\n        with torch.no_grad():\n\n            model.eval()\n            for batch_idx, (text_seqs, popularity_info, labels) in enumerate(tqdm(val_dataloader)):\n                \n                input_ids = text_seqs['input_ids'].to(DEVICE)\n                attention_mask = text_seqs['attention_mask'].to(DEVICE)\n                labels = labels.to(DEVICE)\n                \n                inputs = {\"input_ids\":input_ids, \"attention_mask\":attention_mask}\n                popularity_inputs = popularity_info.to(DEVICE)\n                \n                outputs = model(inputs, popularity_inputs)\n\n                loss = error_func(outputs, labels)\n                \n                val_loss += loss.item()\n\n                # Total correct predictions\n                predicted = torch.sigmoid(outputs).argmax(dim=1)\n                val_correct += (predicted == labels).sum()\n\n\n        train_acc = train_correct / (BATCH_SIZE*len(train_dataloader))\n        val_acc = val_correct / (BATCH_SIZE*len(val_dataloader))\n\n        val_loss = val_loss / len(val_dataloader)\n        scheduler.step(val_loss)\n\n        print(\"# {}\".format(epoch))\n        print('Train Accuracy:{:.3f}%'.format(train_acc*100))\n        print('Validation Accuracy:{:.3f}%'.format(val_acc*100))\n\n        # Save the progress\n\n        modeling_progress['train_acc'].append(train_acc)\n        modeling_progress['val_acc'].append(val_acc)\n        modeling_progress['val_loss'].append(val_loss)\n\n\n        if val_acc > best_val_acc:\n            early_stop_counter = 0\n\n\n            print('Best Val Acc: {:.3f}%'.format(float(val_acc*100)))\n            best_val_acc = val_acc\n            torch.save(model.state_dict(), MODEL_PATH)\n            \n        print(\"=\"*10)\n\n\n        # Save best performance\n        early_stop_counter += 1\n        if early_stop_counter > EARLY_STOP_LIMIT:\n            break\n\n    return modeling_progress","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:50:06.358447Z","iopub.execute_input":"2023-04-19T10:50:06.359224Z","iopub.status.idle":"2023-04-19T10:50:06.376162Z","shell.execute_reply.started":"2023-04-19T10:50:06.359184Z","shell.execute_reply":"2023-04-19T10:50:06.375128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()\nmodeling_progress = fit(model, train_dataloader, val_dataloader)","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:51:00.722756Z","iopub.execute_input":"2023-04-19T10:51:00.723566Z","iopub.status.idle":"2023-04-19T10:51:02.155352Z","shell.execute_reply.started":"2023-04-19T10:51:00.723511Z","shell.execute_reply":"2023-04-19T10:51:02.153836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1,2)\n\n\ntrain_acc_progress = [progress.cpu() for progress in modeling_progress['train_acc']]\nval_acc_progress = [progress.cpu() for progress in modeling_progress['val_acc']]\n\nlen_learning = len(modeling_progress['val_acc'])\nax1.plot(list(range(len_learning)), modeling_progress['val_loss'], label='Validation Loss')\nax1.set_xlabel(\"Epochs\")\nax1.set_ylabel(\"Loss\")\nax1.set_title(\"Loss By Epochs\")\nax1.legend()\n\nax2.plot(list(range(len_learning)), train_acc_progress, label='Training Accuracy')\nax2.plot(list(range(len_learning)), val_acc_progress, label='Validation Accuracy')\nax2.set_xlabel(\"Epochs\")\nax2.set_ylabel(\"Accuracy\")\nax2.set_title(\"Accuracy By Epochs\")\nax2.legend()\n\nfig.set_size_inches(15, 7)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-19T10:51:00.133031Z","iopub.status.idle":"2023-04-19T10:51:00.133818Z","shell.execute_reply.started":"2023-04-19T10:51:00.133542Z","shell.execute_reply":"2023-04-19T10:51:00.133572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"fixed_test_df = preprocess_text(test_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_collate_fn(batch):\n    batch = np.array(batch)\n\n    batch_text_list = batch[:, 0]\n    batch_popularity_list = batch[:, 1:3]  # [votes, comments]\n\n    text_collated = bert_tokenizer(batch_text_list.tolist(), return_tensors='pt', max_length=192, padding='max_length', truncation=True)\n    popularity_collated = torch.LongTensor(batch_popularity_list.astype(np.int32))\n    \n    return text_collated, popularity_collated\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = ReviewDataset(fixed_test_df, is_train=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_dataset)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(\n    dataset=test_dataset, \n    batch_size=BATCH_SIZE, \n    collate_fn=test_collate_fn,\n    shuffle=False,\n    drop_last=False,\n    pin_memory=True,\n    )\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, data_loader):\n    model.eval()\n    y_preds = np.array([])\n    \n    for idx, (text_seqs, popularity_info) in enumerate(tqdm(data_loader)):\n\n        input_ids, attention_mask = text_seqs['input_ids'].to(DEVICE), text_seqs['attention_mask'].to(DEVICE)\n        inputs = {'input_ids':input_ids, 'attention_mask': attention_mask}\n        popularity_inputs = popularity_info.to(DEVICE)\n\n        preds = model(inputs, popularity_inputs)\n\n        preds = torch.sigmoid(preds).argmax(dim=1)\n        preds = preds.cpu()\n        y_preds = np.concatenate((y_preds, preds), axis=0)\n\n    return y_preds\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = LangClassifyModel(num_classes=6, freeze_bert=False)\nmodel2 = model2.to(DEVICE)\nmodel2.load_state_dict(torch.load(MODEL_PATH))\ny_preds = evaluate(model2, test_dataloader)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_preds)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_pd = pd.read_csv('../input/goodreads-books-reviews-290312/goodreads_sample_submission.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_pd['rating'] = y_preds.astype(int)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_pd","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_pd.to_csv(\"sample_submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}