{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-09T15:48:07.681129Z","iopub.execute_input":"2023-01-09T15:48:07.681535Z","iopub.status.idle":"2023-01-09T15:48:07.713374Z","shell.execute_reply.started":"2023-01-09T15:48:07.681435Z","shell.execute_reply":"2023-01-09T15:48:07.712287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:07.715099Z","iopub.execute_input":"2023-01-09T15:48:07.715399Z","iopub.status.idle":"2023-01-09T15:48:08.744864Z","shell.execute_reply.started":"2023-01-09T15:48:07.715366Z","shell.execute_reply":"2023-01-09T15:48:08.743934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Download datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv'\n                      , usecols = ['review_id', 'rating', 'review_text'])\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:11.474928Z","iopub.execute_input":"2023-01-09T15:48:11.475371Z","iopub.status.idle":"2023-01-09T15:48:37.693786Z","shell.execute_reply.started":"2023-01-09T15:48:11.475334Z","shell.execute_reply":"2023-01-09T15:48:37.692807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv'\n                     , usecols = ['review_id', 'review_text'])\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:37.695430Z","iopub.execute_input":"2023-01-09T15:48:37.695698Z","iopub.status.idle":"2023-01-09T15:48:52.903997Z","shell.execute_reply.started":"2023-01-09T15:48:37.695667Z","shell.execute_reply":"2023-01-09T15:48:52.903105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Investigate train dataset a bit","metadata":{}},{"cell_type":"markdown","source":"From Dataset description we know that:\n\n**user_id** - Id of user<br/>\n**book_id** - Id of Book<br/>\n**review_id** - Id of review<br/>\n**rating** - rating from 0 to 5<br/>\n**review_text** - review text<br/>\n**date_added** - date added<br/>\n**date_updated** - date updated<br/>\n**read_at** - read at<br/>\n**started_at** - started at<br/>\n**n_votes** - no. of votes<br/>\n**n_comments** - no. of comments<br/>","metadata":{}},{"cell_type":"code","source":"print(f'Len of train dataset: {len(train_df)}')\nprint(f'Len of test dataset: {len(test_df)}')","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:52.905654Z","iopub.execute_input":"2023-01-09T15:48:52.905985Z","iopub.status.idle":"2023-01-09T15:48:52.911792Z","shell.execute_reply.started":"2023-01-09T15:48:52.905944Z","shell.execute_reply":"2023-01-09T15:48:52.910703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x = train_df.rating)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:52.914534Z","iopub.execute_input":"2023-01-09T15:48:52.914981Z","iopub.status.idle":"2023-01-09T15:48:53.261143Z","shell.execute_reply.started":"2023-01-09T15:48:52.914934Z","shell.execute_reply":"2023-01-09T15:48:53.260223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:53.262545Z","iopub.execute_input":"2023-01-09T15:48:53.263122Z","iopub.status.idle":"2023-01-09T15:48:53.441680Z","shell.execute_reply.started":"2023-01-09T15:48:53.263077Z","shell.execute_reply":"2023-01-09T15:48:53.440720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:53.443123Z","iopub.execute_input":"2023-01-09T15:48:53.443617Z","iopub.status.idle":"2023-01-09T15:48:53.541898Z","shell.execute_reply.started":"2023-01-09T15:48:53.443580Z","shell.execute_reply":"2023-01-09T15:48:53.540933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Clear the data","metadata":{}},{"cell_type":"code","source":"common_df = pd.concat([train_df.review_text, test_df.review_text]).reset_index(drop = True)\ncommon_df","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:53.543232Z","iopub.execute_input":"2023-01-09T15:48:53.543548Z","iopub.status.idle":"2023-01-09T15:48:53.645218Z","shell.execute_reply.started":"2023-01-09T15:48:53.543510Z","shell.execute_reply":"2023-01-09T15:48:53.644377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to lower\ndef to_lower(text):\n    return text.lower()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:53.646499Z","iopub.execute_input":"2023-01-09T15:48:53.646808Z","iopub.status.idle":"2023-01-09T15:48:53.651439Z","shell.execute_reply.started":"2023-01-09T15:48:53.646772Z","shell.execute_reply":"2023-01-09T15:48:53.650387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove stopwords\n#from nltk.corpus import stopwords\n#stop = set(stopwords.words('english'))\nstop = {'a', 'the', 'www', 'http', 'https', 'com'}\n\ndef remove_stopwords(text):\n    return ' '.join([word for word in text.split() if word not in stop])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:53.652626Z","iopub.execute_input":"2023-01-09T15:48:53.652931Z","iopub.status.idle":"2023-01-09T15:48:54.578364Z","shell.execute_reply.started":"2023-01-09T15:48:53.652893Z","shell.execute_reply":"2023-01-09T15:48:54.577638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom urllib.parse import urlparse\n\ndef remove_url(text):\n    url = re.compile(r'https?://\\S+')\n    return url.sub(r'',text)\n\nprint('Example:\\n' + common_df[3])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.582290Z","iopub.execute_input":"2023-01-09T15:48:54.582584Z","iopub.status.idle":"2023-01-09T15:48:54.589337Z","shell.execute_reply.started":"2023-01-09T15:48:54.582549Z","shell.execute_reply":"2023-01-09T15:48:54.588173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\nremove = string.punctuation\n\nperiod = '.'\nremove = remove.replace(period, '')\n\ndef remove_punctuation(text):\n    pattern = re.compile(r\"[{}]\".format(re.escape(remove)))\n    \n    res = []\n    for word in text.split():\n        \n        # remove all punctiations except periods\n        new_word = pattern.sub(r' ', word)\n        new_word = new_word.strip(period)\n        \n        try:\n            float(new_word)\n        except:\n            new_word = new_word.replace(period, ' ')\n            \n        res.append(new_word)\n    \n    return ' '.join(res)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.590773Z","iopub.execute_input":"2023-01-09T15:48:54.591104Z","iopub.status.idle":"2023-01-09T15:48:54.604931Z","shell.execute_reply.started":"2023-01-09T15:48:54.591070Z","shell.execute_reply":"2023-01-09T15:48:54.603880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove all non-alphabetic chars (punctuation, numbers, emojies ...)\ndef remove_non_alphabetic(text):\n    alpha = re.compile(r'[^a-zA-Z]') \n    return alpha.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.606218Z","iopub.execute_input":"2023-01-09T15:48:54.606509Z","iopub.status.idle":"2023-01-09T15:48:54.616670Z","shell.execute_reply.started":"2023-01-09T15:48:54.606478Z","shell.execute_reply":"2023-01-09T15:48:54.615737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_quotes(text):\n    alpha = re.compile(r'\"([^\"\\\\]|\\\\.)*\"') \n    return alpha.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.617771Z","iopub.execute_input":"2023-01-09T15:48:54.618495Z","iopub.status.idle":"2023-01-09T15:48:54.628163Z","shell.execute_reply.started":"2023-01-09T15:48:54.618454Z","shell.execute_reply":"2023-01-09T15:48:54.627372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_spoiler_alert(text):\n    spoiler = re.compile(r'(\\(view spoiler\\).*?\\(hide spoiler\\))') \n    return spoiler.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.629742Z","iopub.execute_input":"2023-01-09T15:48:54.630451Z","iopub.status.idle":"2023-01-09T15:48:54.639435Z","shell.execute_reply.started":"2023-01-09T15:48:54.630413Z","shell.execute_reply":"2023-01-09T15:48:54.638530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_small_words(text):\n    return ' '.join([word for word in text.split() if len(word) > 2])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.640943Z","iopub.execute_input":"2023-01-09T15:48:54.641505Z","iopub.status.idle":"2023-01-09T15:48:54.651019Z","shell.execute_reply.started":"2023-01-09T15:48:54.641470Z","shell.execute_reply":"2023-01-09T15:48:54.650076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LEN = 100","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:48:54.652392Z","iopub.execute_input":"2023-01-09T15:48:54.653149Z","iopub.status.idle":"2023-01-09T15:48:54.661836Z","shell.execute_reply.started":"2023-01-09T15:48:54.653112Z","shell.execute_reply":"2023-01-09T15:48:54.661104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# after applying usual text preprocessing steps (list below)\n# I found ot that many reviews are empty! (because not all of them are meaningful 🤷)\n# so I commented almost all of these steps out to avoid empty reviews in both train and test sets\n\ndef process_text(text):\n    text = to_lower(text)\n    #text = remove_url(text)\n    text = remove_stopwords(text)\n    #text = remove_spoiler_alert(text)\n    #text = remove_quotes(text)\n    #text = remove_non_alphabetic(text)\n    text = remove_punctuation(text)\n    #text = remove_small_words(text)\n    \n    # take only first MAX_LEN words\n    text = ' '.join(text.split()[:MAX_LEN])\n    return text","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:49:13.680794Z","iopub.execute_input":"2023-01-09T15:49:13.681361Z","iopub.status.idle":"2023-01-09T15:49:13.689868Z","shell.execute_reply.started":"2023-01-09T15:49:13.681327Z","shell.execute_reply":"2023-01-09T15:49:13.689061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df = common_df.apply(process_text)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:49:17.476196Z","iopub.execute_input":"2023-01-09T15:49:17.476747Z","iopub.status.idle":"2023-01-09T15:54:19.728991Z","shell.execute_reply.started":"2023-01-09T15:49:17.476695Z","shell.execute_reply":"2023-01-09T15:54:19.727425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check min number of words\ncommon_df.apply(lambda x : len(x.split())).min()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:56:35.122724Z","iopub.execute_input":"2023-01-09T15:56:35.123099Z","iopub.status.idle":"2023-01-09T15:56:40.669820Z","shell.execute_reply.started":"2023-01-09T15:56:35.123060Z","shell.execute_reply":"2023-01-09T15:56:40.668667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(common_df[common_df == ''])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:56:46.947019Z","iopub.execute_input":"2023-01-09T15:56:46.947385Z","iopub.status.idle":"2023-01-09T15:56:47.188897Z","shell.execute_reply.started":"2023-01-09T15:56:46.947339Z","shell.execute_reply":"2023-01-09T15:56:47.187853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if train set has null values \ntrain = common_df.iloc[:len(train_df)].set_axis(train_df.index)\ntrain_df.loc[train[train == ''].index]","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:56:47.191378Z","iopub.execute_input":"2023-01-09T15:56:47.192047Z","iopub.status.idle":"2023-01-09T15:56:47.373487Z","shell.execute_reply.started":"2023-01-09T15:56:47.191997Z","shell.execute_reply":"2023-01-09T15:56:47.372511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if text set has null values \ntest = common_df.iloc[-len(test_df):].set_axis(test_df.index)\ntest_df.loc[test[test == ''].index]","metadata":{"execution":{"iopub.status.busy":"2023-01-09T15:56:47.375174Z","iopub.execute_input":"2023-01-09T15:56:47.375553Z","iopub.status.idle":"2023-01-09T15:56:47.486072Z","shell.execute_reply.started":"2023-01-09T15:56:47.375510Z","shell.execute_reply":"2023-01-09T15:56:47.485257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Lemmatization","metadata":{}},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\nwordnet_lemmatizer = WordNetLemmatizer()\n\ndef lemmatization(text):\n    return ' '.join(wordnet_lemmatizer.lemmatize(word) for word in text.split()) # all words are considered as noun","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:58:35.495762Z","iopub.execute_input":"2023-01-05T22:58:35.496065Z","iopub.status.idle":"2023-01-05T22:58:35.507702Z","shell.execute_reply.started":"2023-01-05T22:58:35.496004Z","shell.execute_reply":"2023-01-05T22:58:35.507083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df = common_df.apply(lemmatization)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:58:35.508768Z","iopub.execute_input":"2023-01-05T22:58:35.509537Z","iopub.status.idle":"2023-01-05T23:06:27.430236Z","shell.execute_reply.started":"2023-01-05T22:58:35.509474Z","shell.execute_reply":"2023-01-05T23:06:27.429004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:27.431466Z","iopub.execute_input":"2023-01-05T23:06:27.431857Z","iopub.status.idle":"2023-01-05T23:06:27.439494Z","shell.execute_reply.started":"2023-01-05T23:06:27.431826Z","shell.execute_reply":"2023-01-05T23:06:27.438196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Target value","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:27.440997Z","iopub.execute_input":"2023-01-05T23:06:27.441506Z","iopub.status.idle":"2023-01-05T23:06:36.289904Z","shell.execute_reply.started":"2023-01-05T23:06:27.441471Z","shell.execute_reply":"2023-01-05T23:06:36.288452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\ny = to_categorical(train_df['rating'])\ny.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:36.294214Z","iopub.execute_input":"2023-01-05T23:06:36.294508Z","iopub.status.idle":"2023-01-05T23:06:36.351998Z","shell.execute_reply.started":"2023-01-05T23:06:36.294477Z","shell.execute_reply":"2023-01-05T23:06:36.350257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Split train set into train and val sets","metadata":{}},{"cell_type":"code","source":"X = common_df.iloc[:y.shape[0]]\nX_test = common_df.iloc[y.shape[0]:]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:36.353428Z","iopub.execute_input":"2023-01-05T23:06:36.353771Z","iopub.status.idle":"2023-01-05T23:06:36.361025Z","shell.execute_reply.started":"2023-01-05T23:06:36.353732Z","shell.execute_reply":"2023-01-05T23:06:36.359418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.15, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:36.362339Z","iopub.execute_input":"2023-01-05T23:06:36.362623Z","iopub.status.idle":"2023-01-05T23:06:36.869799Z","shell.execute_reply.started":"2023-01-05T23:06:36.362588Z","shell.execute_reply":"2023-01-05T23:06:36.865923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Tokenization","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:36.874425Z","iopub.execute_input":"2023-01-05T23:06:36.874793Z","iopub.status.idle":"2023-01-05T23:06:36.882707Z","shell.execute_reply.started":"2023-01-05T23:06:36.874750Z","shell.execute_reply":"2023-01-05T23:06:36.881308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(X_train) # fit tokenizer only on train data\n\nvocab_size = len(tokenizer.word_index)\nvocab_size","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:06:36.884365Z","iopub.execute_input":"2023-01-05T23:06:36.884625Z","iopub.status.idle":"2023-01-05T23:07:41.811795Z","shell.execute_reply.started":"2023-01-05T23:06:36.884596Z","shell.execute_reply":"2023-01-05T23:07:41.810428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sequence = tokenizer.texts_to_sequences(X_train)\ntrain_sequence = pad_sequences(train_sequence, maxlen = MAX_LEN, padding = 'post')\n\nval_sequence = tokenizer.texts_to_sequences(X_val)\nval_sequence = pad_sequences(val_sequence, maxlen = MAX_LEN, padding = 'post')\n\ntest_sequence = tokenizer.texts_to_sequences(X_test)\ntest_sequence = pad_sequences(test_sequence, maxlen = MAX_LEN, padding = 'post')","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:07:41.814505Z","iopub.execute_input":"2023-01-05T23:07:41.814866Z","iopub.status.idle":"2023-01-05T23:09:11.728839Z","shell.execute_reply.started":"2023-01-05T23:07:41.814828Z","shell.execute_reply":"2023-01-05T23:09:11.727634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. TPU Setup","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    strategy = tf.distribute.experimental.TPUStrategy\nexcept ValueError:\n    strategy = tf.distribute.get_strategy() \n    print('Number of replicas:', strategy.num_replicas_in_sync) ","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:09:11.731376Z","iopub.execute_input":"2023-01-05T23:09:11.731816Z","iopub.status.idle":"2023-01-05T23:09:16.713832Z","shell.execute_reply.started":"2023-01-05T23:09:11.731765Z","shell.execute_reply":"2023-01-05T23:09:16.712381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n    \nif tpu:\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu,) \n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nelif len(gpus) > 1:\n    strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n    print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\nelif len(gpus) == 1:\n    strategy = tf.distribute.get_strategy() \n    print('Running on single GPU ', gpus[0].name)\nelse:\n    strategy = tf.distribute.get_strategy() \n    print('Running on CPU')\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:09:16.715853Z","iopub.execute_input":"2023-01-05T23:09:16.716440Z","iopub.status.idle":"2023-01-05T23:09:21.344546Z","shell.execute_reply.started":"2023-01-05T23:09:16.716403Z","shell.execute_reply":"2023-01-05T23:09:21.343292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Conv1D, MaxPooling1D, Embedding, GlobalMaxPooling1D, Flatten, BatchNormalization\nfrom tensorflow.keras.initializers import Constant\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import regularizers\n\n# build model\nwith strategy.scope():\n    model = Sequential()\n    model.add(Embedding(vocab_size + 1, 50, input_length = MAX_LEN))\n\n    model.add(Conv1D(64, 3, activation = 'relu', padding = 'same'))\n    model.add(Conv1D(64, 3, activation = 'relu', padding = 'same'))\n    model.add(MaxPooling1D(2))\n    model.add(Dropout(0.2))\n\n    model.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\n    model.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\n    model.add(MaxPooling1D(3))\n    model.add(Dropout(0.2))\n\n    model.add(Conv1D(256, 5, activation = 'relu', padding = 'same'))\n    model.add(Conv1D(256, 5, activation = 'relu', padding = 'same'))\n    model.add(MaxPooling1D(3))\n    model.add(Dropout(0.2))\n\n    model.add(GlobalMaxPooling1D())\n    model.add(Dense(32, activation = 'relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(y.shape[1], activation = 'softmax'))\n\n    model.compile(Adam(learning_rate = 1e-4),\n                  loss = 'categorical_crossentropy', \n                  metrics=['accuracy'],\n                  steps_per_execution = 200)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:20:02.839558Z","iopub.execute_input":"2023-01-05T23:20:02.839851Z","iopub.status.idle":"2023-01-05T23:20:03.325155Z","shell.execute_reply.started":"2023-01-05T23:20:02.839825Z","shell.execute_reply":"2023-01-05T23:20:03.323706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callbacks\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nearly_stop = EarlyStopping(patience = 4, restore_best_weights = True, verbose = 1) # val_loss\nreduce_lr = ReduceLROnPlateau(factor = 0.1, patience = 2, mode = 'min', verbose = 1) # val_loss","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:12:53.230558Z","iopub.execute_input":"2023-01-05T23:12:53.231272Z","iopub.status.idle":"2023-01-05T23:12:53.237480Z","shell.execute_reply.started":"2023-01-05T23:12:53.231227Z","shell.execute_reply":"2023-01-05T23:12:53.236271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit the model\nhistory = model.fit(train_sequence, \n                    y_train, \n                    batch_size = 64 * strategy.num_replicas_in_sync,\n                    epochs = 25,\n                    verbose = 1,\n                    validation_data = (val_sequence, y_val),\n                    callbacks = [reduce_lr, early_stop])","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:20:08.222933Z","iopub.execute_input":"2023-01-05T23:20:08.223286Z","iopub.status.idle":"2023-01-05T23:21:43.717971Z","shell.execute_reply.started":"2023-01-05T23:20:08.223252Z","shell.execute_reply":"2023-01-05T23:21:43.716456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = history.history['loss']\nacc = history.history['accuracy']\nval_loss = history.history['val_loss']\nval_acc = history.history['val_accuracy']\n\nepochs = range(1, len(loss) + 1)\n\nplt.figure(figsize=(16, 5))\n#accuracy\nplt.subplot(1,2,1)\nplt.plot(epochs, acc, 'bo', label = 'Training accuracy')\nplt.plot(epochs, val_acc, 'r', label = 'Validation accuracy')\nplt.legend()\n\n#loss\nplt.subplot(1,2,2)\nplt.plot(epochs, loss, 'bo', label = 'Trainig loss')\nplt.plot(epochs, val_loss, 'r', label = 'Validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:21:49.173069Z","iopub.execute_input":"2023-01-05T23:21:49.173430Z","iopub.status.idle":"2023-01-05T23:21:49.532082Z","shell.execute_reply.started":"2023-01-05T23:21:49.173393Z","shell.execute_reply":"2023-01-05T23:21:49.531221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Error analysis","metadata":{}},{"cell_type":"code","source":"pred_val = [np.argmax(i) for i in model.predict(val_sequence)]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:21:55.620780Z","iopub.execute_input":"2023-01-05T23:21:55.621318Z","iopub.status.idle":"2023-01-05T23:22:17.045487Z","shell.execute_reply.started":"2023-01-05T23:21:55.621283Z","shell.execute_reply":"2023-01-05T23:22:17.044118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\n\ncolumns = sorted(train_df.rating.unique())\n\ncm = confusion_matrix([np.argmax(i) for i in y_val], pred_val)\ndisp = ConfusionMatrixDisplay(confusion_matrix = cm, display_labels = columns)\ndisp.plot()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:22:19.773557Z","iopub.execute_input":"2023-01-05T23:22:19.773832Z","iopub.status.idle":"2023-01-05T23:22:20.555092Z","shell.execute_reply.started":"2023-01-05T23:22:19.773806Z","shell.execute_reply":"2023-01-05T23:22:20.553715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 11. Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame()\nsub['review_id'] = test_df.review_id\nsub['rating'] = [np.argmax(i) for i in model.predict(test_sequence)]\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"sub.csv\", index = False)","metadata":{},"execution_count":null,"outputs":[]}]}