{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-17T16:57:38.459536Z","iopub.execute_input":"2023-02-17T16:57:38.460028Z","iopub.status.idle":"2023-02-17T16:57:38.504998Z","shell.execute_reply.started":"2023-02-17T16:57:38.459936Z","shell.execute_reply":"2023-02-17T16:57:38.504242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:52:26.263561Z","iopub.execute_input":"2023-01-06T14:52:26.264298Z","iopub.status.idle":"2023-01-06T14:52:27.126429Z","shell.execute_reply.started":"2023-01-06T14:52:26.264261Z","shell.execute_reply":"2023-01-06T14:52:27.125629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Download datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv'\n                      , usecols = ['review_id', 'rating', 'review_text'])\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:52:27.128320Z","iopub.execute_input":"2023-01-06T14:52:27.128670Z","iopub.status.idle":"2023-01-06T14:52:56.345537Z","shell.execute_reply.started":"2023-01-06T14:52:27.128631Z","shell.execute_reply":"2023-01-06T14:52:56.344593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv'\n                     , usecols = ['review_id', 'review_text'])\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:52:56.346672Z","iopub.execute_input":"2023-01-06T14:52:56.346902Z","iopub.status.idle":"2023-01-06T14:53:11.579561Z","shell.execute_reply.started":"2023-01-06T14:52:56.346875Z","shell.execute_reply":"2023-01-06T14:53:11.578740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Investigate train dataset a bit","metadata":{}},{"cell_type":"markdown","source":"From Dataset description we know that:\n\n**user_id** - Id of user<br/>\n**book_id** - Id of Book<br/>\n**review_id** - Id of review<br/>\n**rating** - rating from 0 to 5<br/>\n**review_text** - review text<br/>\n**date_added** - date added<br/>\n**date_updated** - date updated<br/>\n**read_at** - read at<br/>\n**started_at** - started at<br/>\n**n_votes** - no. of votes<br/>\n**n_comments** - no. of comments<br/>","metadata":{}},{"cell_type":"code","source":"print(f'Len of train dataset: {len(train_df)}')\nprint(f'Len of test dataset: {len(test_df)}')","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:11.581749Z","iopub.execute_input":"2023-01-06T14:53:11.582259Z","iopub.status.idle":"2023-01-06T14:53:11.587030Z","shell.execute_reply.started":"2023-01-06T14:53:11.582216Z","shell.execute_reply":"2023-01-06T14:53:11.586411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x = train_df.rating)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:11.588333Z","iopub.execute_input":"2023-01-06T14:53:11.588762Z","iopub.status.idle":"2023-01-06T14:53:11.938865Z","shell.execute_reply.started":"2023-01-06T14:53:11.588732Z","shell.execute_reply":"2023-01-06T14:53:11.937946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:11.940199Z","iopub.execute_input":"2023-01-06T14:53:11.940541Z","iopub.status.idle":"2023-01-06T14:53:12.115446Z","shell.execute_reply.started":"2023-01-06T14:53:11.940474Z","shell.execute_reply":"2023-01-06T14:53:12.114628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:12.116665Z","iopub.execute_input":"2023-01-06T14:53:12.117039Z","iopub.status.idle":"2023-01-06T14:53:12.213280Z","shell.execute_reply.started":"2023-01-06T14:53:12.117009Z","shell.execute_reply":"2023-01-06T14:53:12.212554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Clear the data","metadata":{}},{"cell_type":"code","source":"common_df = pd.concat([train_df.review_text, test_df.review_text]).reset_index(drop = True)\ncommon_df","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:12.214743Z","iopub.execute_input":"2023-01-06T14:53:12.214960Z","iopub.status.idle":"2023-01-06T14:53:12.317090Z","shell.execute_reply.started":"2023-01-06T14:53:12.214935Z","shell.execute_reply":"2023-01-06T14:53:12.316266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to lower\ndef to_lower(text):\n    return text.lower()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:12.320359Z","iopub.execute_input":"2023-01-06T14:53:12.321668Z","iopub.status.idle":"2023-01-06T14:53:12.326139Z","shell.execute_reply.started":"2023-01-06T14:53:12.321632Z","shell.execute_reply":"2023-01-06T14:53:12.325183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove stopwords\n#from nltk.corpus import stopwords\n#stop = set(stopwords.words('english'))\nstop = {'a', 'the', 'www', 'http', 'https', 'com'}\n\ndef remove_stopwords(text):\n    return ' '.join([word for word in text.split() if word not in stop])","metadata":{"execution":{"iopub.status.busy":"2023-02-17T16:57:44.500820Z","iopub.execute_input":"2023-02-17T16:57:44.501460Z","iopub.status.idle":"2023-02-17T16:57:44.507331Z","shell.execute_reply.started":"2023-02-17T16:57:44.501424Z","shell.execute_reply":"2023-02-17T16:57:44.506042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom urllib.parse import urlparse\n\ndef remove_url(text):\n    url = re.compile(r'https?://\\S+')\n    return url.sub(r'',text)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.098843Z","iopub.execute_input":"2023-01-06T14:53:13.099313Z","iopub.status.idle":"2023-01-06T14:53:13.103501Z","shell.execute_reply.started":"2023-01-06T14:53:13.099283Z","shell.execute_reply":"2023-01-06T14:53:13.102927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove all non-alphabetic chars (punctuation, numbers, emojies ...)\ndef remove_non_alphabetic(text):\n    alpha = re.compile(r'[^a-zA-Z]') \n    return alpha.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.104681Z","iopub.execute_input":"2023-01-06T14:53:13.105405Z","iopub.status.idle":"2023-01-06T14:53:13.117316Z","shell.execute_reply.started":"2023-01-06T14:53:13.105371Z","shell.execute_reply":"2023-01-06T14:53:13.116695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\nremove = string.punctuation\n\nperiod = '.'\nremove = remove.replace(period, '')\n\ndef remove_punctuation(text):\n    pattern = re.compile(r\"[{}]\".format(re.escape(remove)))\n    \n    res = []\n    for word in text.split():\n        \n        # remove all punctiations except periods\n        new_word = pattern.sub(r' ', word)\n        new_word = new_word.strip(period)\n        \n        # keep all numbers including float ones in text\n        try:\n            float(new_word)\n        except:\n            new_word = new_word.replace(period, ' ')\n            \n        res.append(new_word)\n    \n    return ' '.join(res)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.118545Z","iopub.execute_input":"2023-01-06T14:53:13.119464Z","iopub.status.idle":"2023-01-06T14:53:13.128792Z","shell.execute_reply.started":"2023-01-06T14:53:13.119427Z","shell.execute_reply":"2023-01-06T14:53:13.128198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_quotes(text):\n    alpha = re.compile(r'\"([^\"\\\\]|\\\\.)*\"') \n    return alpha.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.129868Z","iopub.execute_input":"2023-01-06T14:53:13.130604Z","iopub.status.idle":"2023-01-06T14:53:13.138937Z","shell.execute_reply.started":"2023-01-06T14:53:13.130572Z","shell.execute_reply":"2023-01-06T14:53:13.138342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_spoiler_alert(text):\n    spoiler = re.compile(r'(\\(view spoiler\\).*?\\(hide spoiler\\))') \n    return spoiler.sub(r' ', text)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.140226Z","iopub.execute_input":"2023-01-06T14:53:13.140798Z","iopub.status.idle":"2023-01-06T14:53:13.149309Z","shell.execute_reply.started":"2023-01-06T14:53:13.140759Z","shell.execute_reply":"2023-01-06T14:53:13.148554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_small_words(text):\n    return ' '.join([word for word in text.split() if len(word) > 2])","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.150571Z","iopub.execute_input":"2023-01-06T14:53:13.150997Z","iopub.status.idle":"2023-01-06T14:53:13.158807Z","shell.execute_reply.started":"2023-01-06T14:53:13.150966Z","shell.execute_reply":"2023-01-06T14:53:13.158272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LEN = 100","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.159883Z","iopub.execute_input":"2023-01-06T14:53:13.160384Z","iopub.status.idle":"2023-01-06T14:53:13.168613Z","shell.execute_reply.started":"2023-01-06T14:53:13.160356Z","shell.execute_reply":"2023-01-06T14:53:13.167973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# after applying usual text preprocessing steps (list below)\n# I found ot that many reviews are empty! (because not all of them are meaningful 🤷)\n# so I commented almost all of these steps out to avoid empty reviews in both train and test sets\n\ndef process_text(text):\n    text = to_lower(text)\n    #text = remove_url(text)\n    text = remove_punctuation(text)\n    #text = remove_spoiler_alert(text)\n    #text = remove_quotes(text)\n    #text = remove_non_alphabetic(text)\n    #text = remove_stopwords(text)\n    #text = remove_small_words(text)\n    \n    # take only first MAX_LEN words\n    text = ' '.join(text.split()[:MAX_LEN])\n    return text","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.169921Z","iopub.execute_input":"2023-01-06T14:53:13.170250Z","iopub.status.idle":"2023-01-06T14:53:13.179053Z","shell.execute_reply.started":"2023-01-06T14:53:13.170218Z","shell.execute_reply":"2023-01-06T14:53:13.178467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df = common_df.apply(process_text)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T14:53:13.179920Z","iopub.execute_input":"2023-01-06T14:53:13.180574Z","iopub.status.idle":"2023-01-06T15:00:44.852937Z","shell.execute_reply.started":"2023-01-06T14:53:13.180544Z","shell.execute_reply":"2023-01-06T15:00:44.852020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Lemmatization","metadata":{}},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\nwordnet_lemmatizer = WordNetLemmatizer()\n\ndef lemmatization(text):\n    return ' '.join(wordnet_lemmatizer.lemmatize(word) for word in text.split())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df = common_df.apply(lemmatization)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_df[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Target value","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-01-06T15:00:44.854182Z","iopub.execute_input":"2023-01-06T15:00:44.854493Z","iopub.status.idle":"2023-01-06T15:00:50.686181Z","shell.execute_reply.started":"2023-01-06T15:00:44.854454Z","shell.execute_reply":"2023-01-06T15:00:50.685317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = tf.keras.utils.to_categorical(train_df['rating'])\ny.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-06T15:00:50.687306Z","iopub.execute_input":"2023-01-06T15:00:50.687552Z","iopub.status.idle":"2023-01-06T15:00:50.729282Z","shell.execute_reply.started":"2023-01-06T15:00:50.687525Z","shell.execute_reply":"2023-01-06T15:00:50.728392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Split train set into train and val sets","metadata":{}},{"cell_type":"code","source":"X = common_df.iloc[:y.shape[0]]\nX_test = common_df.iloc[y.shape[0]:]","metadata":{"execution":{"iopub.status.busy":"2023-01-06T15:00:50.730499Z","iopub.execute_input":"2023-01-06T15:00:50.730772Z","iopub.status.idle":"2023-01-06T15:00:50.738706Z","shell.execute_reply.started":"2023-01-06T15:00:50.730741Z","shell.execute_reply":"2023-01-06T15:00:50.737723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.15, random_state = 42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. TPU Setup","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    strategy = tf.distribute.experimental.TPUStrategy\nexcept ValueError:\n    strategy = tf.distribute.get_strategy() \n    print('Number of replicas:', strategy.num_replicas_in_sync) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n    \nif tpu:\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu,) \n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nelif len(gpus) > 1:\n    strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n    print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\nelif len(gpus) == 1:\n    strategy = tf.distribute.get_strategy() \n    print('Running on single GPU ', gpus[0].name)\nelse:\n    strategy = tf.distribute.get_strategy() \n    print('Running on CPU')\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Tokenization","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(X_train) # fit tokenizer only on train data\n\nvocab_size = len(tokenizer.word_index)\nvocab_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sequence = tokenizer.texts_to_sequences(X_train)\ntrain_sequence = pad_sequences(train_sequence, maxlen = MAX_LEN, padding = 'post')\n\nval_sequence = tokenizer.texts_to_sequences(X_val)\nval_sequence = pad_sequences(val_sequence, maxlen = MAX_LEN, padding = 'post')\n\ntest_sequence = tokenizer.texts_to_sequences(X_test)\ntest_sequence = pad_sequences(test_sequence, maxlen = MAX_LEN, padding = 'post')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. GloVe","metadata":{}},{"cell_type":"code","source":"# download GvoVe dict (word: [vector representation])\n\nembedding_dict = {}\nwith open('../input/glove-global-vectors-for-word-representation/glove.6B.50d.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:], 'float32')\n        embedding_dict[word] = vectors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size_glove = vocab_size + 1\nembedding_matrix = np.zeros((vocab_size_glove, 50)) # glove.6B.100d.txt\n\nfor word, i in tokenizer.word_index.items():\n    if i > vocab_size_glove:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Conv1D, MaxPooling1D, Embedding, GlobalMaxPooling1D, Flatten\nfrom tensorflow.keras.initializers import Constant\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import regularizers\n\n\nmodel = Sequential()\nmodel.add(Embedding(vocab_size_glove, \n                    50, \n                    input_length = MAX_LEN,\n                    embeddings_initializer = Constant(embedding_matrix), \n                    trainable = False))\n\nmodel.add(Conv1D(64, 3, activation = 'relu', padding = 'same'))\nmodel.add(Conv1D(64, 3, activation = 'relu', padding = 'same'))\nmodel.add(MaxPooling1D(2))\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\nmodel.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\nmodel.add(MaxPooling1D(3))\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\nmodel.add(Conv1D(128, 5, activation = 'relu', padding = 'same'))\nmodel.add(MaxPooling1D(3))\nmodel.add(Dropout(0.3))\n\nmodel.add(GlobalMaxPooling1D())\n\nmodel.add(Dense(32, activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(y.shape[1], activation = 'softmax'))\n\nmodel.compile(Adam(learning_rate = 1e-3),\n              loss = 'categorical_crossentropy', \n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callbacks\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nearly_stop = EarlyStopping(patience = 5, restore_best_weights = True, verbose = 1) # val_loss\nreduce_lr = ReduceLROnPlateau(factor = 0.1, patience = 2, mode = 'min', verbose = 1) # val_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit the model\nhistory = model.fit(train_sequence, \n                    y_train, \n                    batch_size = 128,\n                    epochs = 25,\n                    verbose = 1, \n                    validation_data = (val_sequence, y_val),\n                    callbacks = [reduce_lr, early_stop])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = history.history['loss']\nacc = history.history['accuracy']\nval_loss = history.history['val_loss']\nval_acc = history.history['val_accuracy']\n\nepochs = range(1, len(loss) + 1)\n\nplt.figure(figsize=(16, 5))\n#accuracy\nplt.subplot(1,2,1)\nplt.plot(epochs, acc, 'bo', label = 'Training accuracy')\nplt.plot(epochs, val_acc, 'r', label = 'Validation accuracy')\nplt.legend()\n\n#loss\nplt.subplot(1,2,2)\nplt.plot(epochs, loss, 'bo', label = 'Trainig loss')\nplt.plot(epochs, val_loss, 'r', label = 'Validation loss')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 11. Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame()\nsub['review_id'] = test_df.review_id\nsub['rating'] = [np.argmax(i) for i in model.predict(test_sequence)]\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"sub.csv\", index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}