{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-09T07:06:41.224716Z","iopub.execute_input":"2021-07-09T07:06:41.225111Z","iopub.status.idle":"2021-07-09T07:07:04.513376Z","shell.execute_reply.started":"2021-07-09T07:06:41.225016Z","shell.execute_reply":"2021-07-09T07:07:04.512127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\nfrom itertools import chain\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\nfrom pathlib import Path\nimport collections\nimport pickle\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport en_core_web_lg\nimport spacy\nfrom spacy.lang.en import English\nfrom spacy.util import minibatch, compounding\nfrom spacy import displacy\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:04.515889Z","iopub.execute_input":"2021-07-09T07:07:04.516473Z","iopub.status.idle":"2021-07-09T07:07:15.300744Z","shell.execute_reply.started":"2021-07-09T07:07:04.516425Z","shell.execute_reply":"2021-07-09T07:07:15.299574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions to Prepare Text","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.303318Z","iopub.execute_input":"2021-07-09T07:07:15.303743Z","iopub.status.idle":"2021-07-09T07:07:15.316184Z","shell.execute_reply.started":"2021-07-09T07:07:15.303699Z","shell.execute_reply":"2021-07-09T07:07:15.315039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\n\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.318764Z","iopub.execute_input":"2021-07-09T07:07:15.319396Z","iopub.status.idle":"2021-07-09T07:07:15.343006Z","shell.execute_reply.started":"2021-07-09T07:07:15.319349Z","shell.execute_reply":"2021-07-09T07:07:15.341991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add callbacks\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\n\n#define the callbacks\nearly_stopping = [EarlyStopping(monitor = 'val_loss', patience = 5, verbose = 1),\n                 ModelCheckpoint(filepath = 'gru_model.h5', monitor = 'val_loss', save_best_only = True)]","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.344514Z","iopub.execute_input":"2021-07-09T07:07:15.345158Z","iopub.status.idle":"2021-07-09T07:07:15.353768Z","shell.execute_reply.started":"2021-07-09T07:07:15.345087Z","shell.execute_reply":"2021-07-09T07:07:15.352656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.355578Z","iopub.execute_input":"2021-07-09T07:07:15.356025Z","iopub.status.idle":"2021-07-09T07:07:15.361839Z","shell.execute_reply.started":"2021-07-09T07:07:15.355946Z","shell.execute_reply":"2021-07-09T07:07:15.360231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.363936Z","iopub.execute_input":"2021-07-09T07:07:15.364967Z","iopub.status.idle":"2021-07-09T07:07:15.376430Z","shell.execute_reply.started":"2021-07-09T07:07:15.364918Z","shell.execute_reply":"2021-07-09T07:07:15.375056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n    \ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.380824Z","iopub.execute_input":"2021-07-09T07:07:15.381306Z","iopub.status.idle":"2021-07-09T07:07:15.390370Z","shell.execute_reply.started":"2021-07-09T07:07:15.381268Z","shell.execute_reply":"2021-07-09T07:07:15.389268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.393317Z","iopub.execute_input":"2021-07-09T07:07:15.393827Z","iopub.status.idle":"2021-07-09T07:07:15.403344Z","shell.execute_reply.started":"2021-07-09T07:07:15.393781Z","shell.execute_reply":"2021-07-09T07:07:15.401584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DEEP LEARNING: GRU\n\n* Tokenizes the texts into words\n* Creates a vocabulary using the top 20,000 tokens\n* Converts the tokens into sequence vectors\n* Pads the sequences to a fixed sequence length","metadata":{}},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"code","source":"#read \ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\n#review\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.405866Z","iopub.execute_input":"2021-07-09T07:07:15.406550Z","iopub.status.idle":"2021-07-09T07:07:15.548428Z","shell.execute_reply.started":"2021-07-09T07:07:15.406504Z","shell.execute_reply":"2021-07-09T07:07:15.547157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\n\nDATA = []\nlabel_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n\n    balanced = False\n    \n    sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(data))]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    for sentence in sentences:          \n      \n        a = re.search(row.cleaned_label.lower(), sentence)      \n        b = re.search(row.dataset_label.lower(), sentence)\n        c = re.search(row.dataset_title.lower(), sentence)\n        cleaned_label = row.cleaned_label.lower()\n        dataset_label = row.dataset_label.lower()\n        dataset_title = row.dataset_title.lower()\n        \n        if  a != None:\n            DATA.append((sentence, cleaned_label))\n            label_count = label_count + 1\n            balanced = True\n        elif b != None:\n            DATA.append((sentence, dataset_label))\n            label_count = label_count + 1\n            balanced = True\n        elif c != None:\n            DATA.append((sentence, dataset_title))\n            label_count = label_count + 1\n            balanced = True            \n        else:\n            if balanced:\n                empty_count = empty_count + 1\n                balanced = False\n    \nprint('Text with dataset:', label_count)\nprint('Text without dataset:', empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:07:15.549987Z","iopub.execute_input":"2021-07-09T07:07:15.550419Z","iopub.status.idle":"2021-07-09T07:31:41.401620Z","shell.execute_reply.started":"2021-07-09T07:07:15.550375Z","shell.execute_reply":"2021-07-09T07:31:41.400279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\ntrain_df = pd.DataFrame(DATA)\ntrain_df = train_df.rename({0: 'Sentence', 1: 'Label'}, axis = 1)\n\n#review\ntrain_df.tail(10)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.403383Z","iopub.execute_input":"2021-07-09T07:31:41.404147Z","iopub.status.idle":"2021-07-09T07:31:41.439651Z","shell.execute_reply.started":"2021-07-09T07:31:41.404104Z","shell.execute_reply":"2021-07-09T07:31:41.438160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][1000])\nprint('\\n')\nprint(train_df['Label'][1000])","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.441386Z","iopub.execute_input":"2021-07-09T07:31:41.441842Z","iopub.status.idle":"2021-07-09T07:31:41.449564Z","shell.execute_reply.started":"2021-07-09T07:31:41.441796Z","shell.execute_reply":"2021-07-09T07:31:41.448247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][15689])\nprint('\\n')\nprint(train_df['Label'][15678])","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.451493Z","iopub.execute_input":"2021-07-09T07:31:41.452274Z","iopub.status.idle":"2021-07-09T07:31:41.463229Z","shell.execute_reply.started":"2021-07-09T07:31:41.452230Z","shell.execute_reply":"2021-07-09T07:31:41.461785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][23456])\nprint('\\n')\nprint(train_df['Label'][23456])","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.465181Z","iopub.execute_input":"2021-07-09T07:31:41.465673Z","iopub.status.idle":"2021-07-09T07:31:41.474649Z","shell.execute_reply.started":"2021-07-09T07:31:41.465623Z","shell.execute_reply":"2021-07-09T07:31:41.473330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"### Train-Test-Split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX = train_df['Sentence'].to_numpy()\ny = train_df['Label'].to_numpy()\n\n#split traing data into training a validation sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 123)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.476589Z","iopub.execute_input":"2021-07-09T07:31:41.477458Z","iopub.status.idle":"2021-07-09T07:31:41.493275Z","shell.execute_reply.started":"2021-07-09T07:31:41.477412Z","shell.execute_reply":"2021-07-09T07:31:41.492195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check shape\nprint('Train sentences:', X_train.shape, '\\n', \n      'Test sentences:', X_test.shape, '\\n', \n      'Train labels:', y_train.shape, '\\n', \n      'Test labels:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.494964Z","iopub.execute_input":"2021-07-09T07:31:41.495446Z","iopub.status.idle":"2021-07-09T07:31:41.507426Z","shell.execute_reply.started":"2021-07-09T07:31:41.495400Z","shell.execute_reply":"2021-07-09T07:31:41.505876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenize Features","metadata":{}},{"cell_type":"code","source":"#https://developers.google.com/machine-learning/guides/text-classification/step-3\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#limit on the number of features. We use the top 20K features\ntop_k = 20000\n\n#limit on the length of text sequences. Sequences longer than this will be truncated\nmax_sequence_length = 100\n\n#get max sequence length\nmax_length = len(max(X_train, key = len))\nif max_length > max_sequence_length:\n    max_length = max_sequence_length\n    \n#max_vocab_length = 20000 # max number of words to have in our vocabulary\n\n#method to count the unique words in vocabulary and assign each of those words to indices\ntokenizer = Tokenizer(num_words = top_k)\n\n#create vocabulary with training texts\ntokenizer.fit_on_texts(list(X_train))\n\n#convert text into integer sequences\nX_train_seq = tokenizer.texts_to_sequences(X_train)\nX_test_seq = tokenizer.texts_to_sequences(X_test)\n         \n#fix sequence length to max value. \n#sequences shorter than the length are padded in the beginning and sequences longer are truncated at the beginning\n#this turns our lists of integers into a 2D integer tensor of shape (samples, maxlen)\nX_train_pad  = pad_sequences(X_train_seq, maxlen = max_length)\nX_test_pad = pad_sequences(X_test_seq, maxlen = max_length)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:41.509502Z","iopub.execute_input":"2021-07-09T07:31:41.510435Z","iopub.status.idle":"2021-07-09T07:31:46.710264Z","shell.execute_reply.started":"2021-07-09T07:31:41.510334Z","shell.execute_reply":"2021-07-09T07:31:46.709189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check shape\nprint('Train sentences:', X_train_pad.shape, '\\n', \n      'Test sentences:', X_test_pad.shape, '\\n', \n      'Train labels:', y_train.shape, '\\n', \n      'Test labels:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.711789Z","iopub.execute_input":"2021-07-09T07:31:46.712283Z","iopub.status.idle":"2021-07-09T07:31:46.725850Z","shell.execute_reply.started":"2021-07-09T07:31:46.712241Z","shell.execute_reply":"2021-07-09T07:31:46.723435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.727980Z","iopub.execute_input":"2021-07-09T07:31:46.728908Z","iopub.status.idle":"2021-07-09T07:31:46.739160Z","shell.execute_reply.started":"2021-07-09T07:31:46.728835Z","shell.execute_reply":"2021-07-09T07:31:46.737633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_pad","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.741140Z","iopub.execute_input":"2021-07-09T07:31:46.742024Z","iopub.status.idle":"2021-07-09T07:31:46.753592Z","shell.execute_reply.started":"2021-07-09T07:31:46.741706Z","shell.execute_reply":"2021-07-09T07:31:46.751909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_pad","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.755792Z","iopub.execute_input":"2021-07-09T07:31:46.756378Z","iopub.status.idle":"2021-07-09T07:31:46.765440Z","shell.execute_reply.started":"2021-07-09T07:31:46.756333Z","shell.execute_reply":"2021-07-09T07:31:46.763811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of unique words in the training data\nsize_of_vocabulary = len(tokenizer.word_index) + 1 #+1 for padding\nprint(size_of_vocabulary)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.771211Z","iopub.execute_input":"2021-07-09T07:31:46.771991Z","iopub.status.idle":"2021-07-09T07:31:46.779571Z","shell.execute_reply.started":"2021-07-09T07:31:46.771942Z","shell.execute_reply":"2021-07-09T07:31:46.778158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get word index\nword_index = tokenizer.word_index","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.781995Z","iopub.execute_input":"2021-07-09T07:31:46.782852Z","iopub.status.idle":"2021-07-09T07:31:46.795655Z","shell.execute_reply.started":"2021-07-09T07:31:46.782805Z","shell.execute_reply":"2021-07-09T07:31:46.794226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get number of distinct characters\nmax_id = len(tokenizer.word_index)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.797375Z","iopub.execute_input":"2021-07-09T07:31:46.797857Z","iopub.status.idle":"2021-07-09T07:31:46.806491Z","shell.execute_reply.started":"2021-07-09T07:31:46.797811Z","shell.execute_reply":"2021-07-09T07:31:46.805368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encode Label","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#use the LabelEncoder to convert text labels to integers, 0, 1, 2, etc.\nencoder = preprocessing.LabelEncoder()\n\n#since we have two different data set (X_train and X_test), \n#we need to fit it on all of our data otherwise there might be some categories in the test set X_test that were not in the train set X_train \n#and we will get errors\nencoder.fit(list(y_train) + list(y_test)) \ny_train = encoder.transform(y_train)\ny_test = encoder.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.808386Z","iopub.execute_input":"2021-07-09T07:31:46.809184Z","iopub.status.idle":"2021-07-09T07:31:46.985231Z","shell.execute_reply.started":"2021-07-09T07:31:46.809136Z","shell.execute_reply":"2021-07-09T07:31:46.984053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Binarize Label","metadata":{}},{"cell_type":"code","source":"num_classes = train_df['Label'].nunique() + 1\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:46.987020Z","iopub.execute_input":"2021-07-09T07:31:46.987500Z","iopub.status.idle":"2021-07-09T07:31:47.003510Z","shell.execute_reply.started":"2021-07-09T07:31:46.987453Z","shell.execute_reply":"2021-07-09T07:31:47.001832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import utils\n\n#binarize the labels for the neural net\ny_train = utils.to_categorical(y_train, num_classes)\ny_test = utils.to_categorical(y_test, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:47.005528Z","iopub.execute_input":"2021-07-09T07:31:47.006263Z","iopub.status.idle":"2021-07-09T07:31:47.030905Z","shell.execute_reply.started":"2021-07-09T07:31:47.006215Z","shell.execute_reply":"2021-07-09T07:31:47.029595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:47.032615Z","iopub.execute_input":"2021-07-09T07:31:47.033109Z","iopub.status.idle":"2021-07-09T07:31:47.041374Z","shell.execute_reply.started":"2021-07-09T07:31:47.033064Z","shell.execute_reply":"2021-07-09T07:31:47.039954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2021-07-09T07:31:47.043342Z","iopub.execute_input":"2021-07-09T07:31:47.044188Z","iopub.status.idle":"2021-07-09T07:31:47.054722Z","shell.execute_reply.started":"2021-07-09T07:31:47.044113Z","shell.execute_reply":"2021-07-09T07:31:47.053139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GRU with GloVe Embedding\n\nGlobal Vectors for Word Representation, or GloVe, is an “unsupervised learning algorithm for obtaining vector representations for words.” Simply put, GloVe allows us to take a corpus of text, and intuitively transform each word in that corpus into a position in a high-dimensional space. This means that similar words will be placed together.","metadata":{}},{"cell_type":"code","source":"#load the whole embedding into memory\nembeddings_index = {}\nf = open('../input/glove840b300dtxt/glove.840B.300d.txt')\n\nfor line in tqdm(f):\n    values = line.split()\n    word = values[0]\n    try:\n        coefs = np.asarray(values[1:], dtype = 'float32')\n        embeddings_index[word] = coefs\n    except ValueError: #catch the exception where there are strings in the GloVe text file, can be avoided if use glove.42B.300d.txt\n        pass\n    \nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:25:28.960607Z","iopub.execute_input":"2021-07-08T08:25:28.961227Z","iopub.status.idle":"2021-07-08T08:30:33.535068Z","shell.execute_reply.started":"2021-07-08T08:25:28.96111Z","shell.execute_reply":"2021-07-08T08:30:33.533768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:33.536907Z","iopub.execute_input":"2021-07-08T08:30:33.537427Z","iopub.status.idle":"2021-07-08T08:30:33.744418Z","shell.execute_reply.started":"2021-07-08T08:30:33.53738Z","shell.execute_reply":"2021-07-08T08:30:33.743062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Words in a given dataset are most likely not unique to that dataset. We can thus learn the relationship between the words in our dataset using other dataset(s). To do so, we can transfer an embedding learned from another dataset into our embedding layer. These embeddings are referred to as pre-trained embeddings. Using a pre-trained embedding gives the model a head start in the learning process.\n\nThere are pre-trained embeddings available that have been trained using large corpora, such as GloVe. GloVe has been trained on multiple corpora (primarily Wikipedia). ","metadata":{}},{"cell_type":"code","source":"nonzero_elements = np.count_nonzero(np.count_nonzero(embedding_matrix, axis=1))\nnonzero_elements / size_of_vocabulary","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:33.746205Z","iopub.execute_input":"2021-07-08T08:30:33.746785Z","iopub.status.idle":"2021-07-08T08:30:33.757161Z","shell.execute_reply.started":"2021-07-08T08:30:33.746725Z","shell.execute_reply":"2021-07-08T08:30:33.755656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GRU","metadata":{}},{"cell_type":"code","source":"GRU_model = Sequential()\n\n#embedding layer\nGRU_model.add(Embedding(size_of_vocabulary, 300,\n                        weights = [embedding_matrix],\n                        input_length = X_train_pad.shape[0],\n                        trainable = False)) \n\n#model architecture\nGRU_model.add(SpatialDropout1D(0.3))\nGRU_model.add(GRU(300, dropout = 0.3, recurrent_dropout = 0.3, return_sequences = True))\nGRU_model.add(GRU(300, dropout = 0.3, recurrent_dropout = 0.3))\n\n#fully connected layers\nGRU_model.add(Dense(512, activation = 'relu'))\nGRU_model.add(Dropout(0.3))\n\nGRU_model.add(Dense(128, activation = 'relu')) \nGRU_model.add(Dropout(0.3))\n\nGRU_model.add(Dense(64, activation = 'relu')) \nGRU_model.add(Dropout(0.3))\n\n#output layer\nGRU_model.add(Dense(num_classes, activation = 'softmax')) ","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:33.759015Z","iopub.execute_input":"2021-07-08T08:30:33.759688Z","iopub.status.idle":"2021-07-08T08:30:37.123816Z","shell.execute_reply.started":"2021-07-08T08:30:33.759641Z","shell.execute_reply":"2021-07-08T08:30:37.122651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary\nGRU_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:37.12555Z","iopub.execute_input":"2021-07-08T08:30:37.125975Z","iopub.status.idle":"2021-07-08T08:30:37.147019Z","shell.execute_reply.started":"2021-07-08T08:30:37.125933Z","shell.execute_reply":"2021-07-08T08:30:37.144597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nplot_model(GRU_model, to_file = 'GRU_model_plot.png', show_shapes = True, show_layer_names = True)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:37.148487Z","iopub.execute_input":"2021-07-08T08:30:37.149106Z","iopub.status.idle":"2021-07-08T08:30:37.894696Z","shell.execute_reply.started":"2021-07-08T08:30:37.14906Z","shell.execute_reply":"2021-07-08T08:30:37.887851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compile\nGRU_model.compile(optimizer = keras.optimizers.Adam(0.0001), \n                    loss = 'categorical_crossentropy',\n                    metrics = ['acc']) ","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:37.896655Z","iopub.execute_input":"2021-07-08T08:30:37.89712Z","iopub.status.idle":"2021-07-08T08:30:37.922645Z","shell.execute_reply.started":"2021-07-08T08:30:37.897075Z","shell.execute_reply":"2021-07-08T08:30:37.921306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.datetime.now()\nbatch_size = 256\n\n#fit\nGRU_history = GRU_model.fit(np.array(X_train_pad), np.array(y_train),\n                            #class_weight = weights, \n                            batch_size = batch_size,\n                            epochs = 30,\n                            validation_data = (np.array(X_test_pad), np.array(y_test)),\n                            steps_per_epoch = X_train_pad.shape[0] // 256,\n                            validation_steps = X_test_pad.shape[0] // 256,\n                            verbose = 1,\n                            callbacks = early_stopping)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T08:30:37.924902Z","iopub.execute_input":"2021-07-08T08:30:37.925521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = datetime.datetime.now()\nelapsed = end - start\nprint('Training took a total of {}'.format(elapsed))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model\nGRU_model.save('gru_model.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Evaluation","metadata":{}},{"cell_type":"code","source":"fig , ax = plt.subplots(1,2)\nfig.set_size_inches(20, 8)\n\nGRU_train_acc = GRU_history.history['acc']\nGRU_train_loss = GRU_history.history['loss']\nGRU_val_acc = GRU_history.history['val_acc']\nGRU_val_loss = GRU_history.history['val_loss']\n\nepochs = range(1, len(GRU_train_acc) + 1)\n\nax[0].plot(epochs , GRU_train_acc , 'g-o' , label = 'Training Accuracy')\nax[0].plot(epochs , GRU_val_acc , 'y-o' , label = 'Validation Accuracy')\nax[0].set_title('GRU Model Train & Validation Accuracy')\nax[0].legend(loc = 'lower right')\nax[0].set_xlabel('Epochs')\nax[0].set_ylabel('Accuracy')\n\nax[1].plot(epochs, GRU_train_loss , 'g-o' , label = 'Training Loss')\nax[1].plot(epochs, GRU_val_loss , 'y-o' , label = 'Validation Loss')\nax[1].set_title('GRU Model Train & Validation Loss')\nax[1].legend()\nax[1].set_xlabel('Epochs')\nax[1].set_ylabel('Accuracy')\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nplt.savefig('gru_acc_loss.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train loss & accuracy:', GRU_model.evaluate(X_train_pad, y_train))\nprint('\\n')\nprint('Test loss & accuracy:', GRU_model.evaluate(X_test_pad, y_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make prediction\nGRU_yhat_test = GRU_model.predict(X_test_pad)\n\n#to evaluate accuracy we need a vector of labels\nGRU_yhat_test = np.argmax(GRU_yhat_test, axis = 1)\nGRU_y_test = np.argmax(y_test, axis = 1)\n\n#get classification report\nprint('Model: GRU', '\\n', classification_report(GRU_y_test, GRU_yhat_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary table\nsummary_table = pd.DataFrame({'Model': [],\n                              'Accuracy': [],\n                              'Precision': [], 'Recall': [], 'F1': []})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#update summary table\nsummary_table.loc[0] = ['RNN GRU',\n                        round(accuracy_score(GRU_y_test, GRU_yhat_test), 2),\n                        round(precision_score(GRU_y_test, GRU_yhat_test, average = 'macro'), 2), \n                        round(recall_score(GRU_y_test, GRU_yhat_test, average = 'macro'), 2), \n                        round(f1_score(GRU_y_test, GRU_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsummary_table.to_csv('gru_summary_table.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"#get text\ntqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_json_pub)\n\n#clean text\ntrain['text'] = train['text'].progress_apply(clean_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\n#read data\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(read_json_pub, train_path = test_path))\n\n#review\nsample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_matching = True\ngru_prediction = True\n\nid_list = []\nlabels_list = []\n\nfor index, row in tqdm(sample_submission.iterrows()):\n\n    sample_text = row['text']\n\n    row_id = row['Id']\n    \n    #takes only the rows where train file is identical to a test file\n    temp_df = train[train['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    #literal_matching \n    if literal_matching:\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():    \n                cleaned_labels.append(clean_text(known_label))\n            \n        print('cleaned label:', set(cleaned_labels))   \n    \n    #gru_prediction \n    if gru_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n        \n        tokenizer.fit_on_texts(list(sentence))\n        sentence_seq = tokenizer.texts_to_sequences([sentence])\n        sentence_pad  = pad_sequences(sentence_seq, maxlen = max_length)\n            \n        #predict\n        gru_labels = GRU_model.predict(sentence_pad)\n    \n        #get label\n        gru_labels = encoder.inverse_transform([np.argmax(gru_labels)])\n        print('gru label:', set(gru_labels))\n        gru_labels = set(gru_labels)\n        \n    cleaned_labels += gru_labels\n        \n    cleaned_labels = set(cleaned_labels)\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]    \n    labels_list.append('|'.join(cleaned_labels))\n    print('label list:', labels_list)   \n    id_list.append(row_id)\n    print('\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\nsample_submission['PredictionString'] = labels_list\nsample_submission.drop(columns = 'text', axis = 1, inplace = True)\nsample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission['PredictionString'][0])\nprint('\\n')\nprint(sample_submission['PredictionString'][1])\nprint('\\n')\nprint(sample_submission['PredictionString'][2])\nprint('\\n')\nprint(sample_submission['PredictionString'][3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsample_submission.to_csv('submission.csv', index = False)\n\n#check\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport psutil\nprocess = psutil.Process(os.getpid())\nprint(process.memory_percent())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}