{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T01:34:58.166406Z","iopub.execute_input":"2021-06-21T01:34:58.166969Z","iopub.status.idle":"2021-06-21T01:35:11.965423Z","shell.execute_reply.started":"2021-06-21T01:34:58.166878Z","shell.execute_reply":"2021-06-21T01:35:11.964582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\n\nimport spacy\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:11.966832Z","iopub.execute_input":"2021-06-21T01:35:11.967129Z","iopub.status.idle":"2021-06-21T01:35:22.158204Z","shell.execute_reply.started":"2021-06-21T01:35:11.967099Z","shell.execute_reply":"2021-06-21T01:35:22.156972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\n\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.160655Z","iopub.execute_input":"2021-06-21T01:35:22.161097Z","iopub.status.idle":"2021-06-21T01:35:22.181475Z","shell.execute_reply.started":"2021-06-21T01:35:22.161043Z","shell.execute_reply":"2021-06-21T01:35:22.180242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add callbacks\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\n\n#define the callbacks\nearly_stopping = [EarlyStopping(monitor = 'val_loss', patience = 5, verbose = 1),\n                 ModelCheckpoint(filepath = 'lstm_model.h5', monitor = 'val_loss', save_best_only = True)]","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.183051Z","iopub.execute_input":"2021-06-21T01:35:22.183614Z","iopub.status.idle":"2021-06-21T01:35:22.187880Z","shell.execute_reply.started":"2021-06-21T01:35:22.183578Z","shell.execute_reply":"2021-06-21T01:35:22.187175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.189034Z","iopub.execute_input":"2021-06-21T01:35:22.189441Z","iopub.status.idle":"2021-06-21T01:35:22.203790Z","shell.execute_reply.started":"2021-06-21T01:35:22.189412Z","shell.execute_reply":"2021-06-21T01:35:22.202634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.205432Z","iopub.execute_input":"2021-06-21T01:35:22.205734Z","iopub.status.idle":"2021-06-21T01:35:22.218225Z","shell.execute_reply.started":"2021-06-21T01:35:22.205704Z","shell.execute_reply":"2021-06-21T01:35:22.217370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n    \ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.219852Z","iopub.execute_input":"2021-06-21T01:35:22.220559Z","iopub.status.idle":"2021-06-21T01:35:22.234032Z","shell.execute_reply.started":"2021-06-21T01:35:22.220502Z","shell.execute_reply":"2021-06-21T01:35:22.232931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.237171Z","iopub.execute_input":"2021-06-21T01:35:22.237589Z","iopub.status.idle":"2021-06-21T01:35:22.254754Z","shell.execute_reply.started":"2021-06-21T01:35:22.237552Z","shell.execute_reply":"2021-06-21T01:35:22.253601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.256652Z","iopub.execute_input":"2021-06-21T01:35:22.257011Z","iopub.status.idle":"2021-06-21T01:35:22.267272Z","shell.execute_reply.started":"2021-06-21T01:35:22.256974Z","shell.execute_reply":"2021-06-21T01:35:22.266206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read \ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\n#review\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.268463Z","iopub.execute_input":"2021-06-21T01:35:22.268944Z","iopub.status.idle":"2021-06-21T01:35:22.467882Z","shell.execute_reply.started":"2021-06-21T01:35:22.268903Z","shell.execute_reply":"2021-06-21T01:35:22.466623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Sentences & Labels","metadata":{}},{"cell_type":"code","source":"import nltk\n\nDATA = []\nlabel_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n\n    balanced = False\n    \n    sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(data))]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n        \n    for sentence in sentences:          \n     \n        a = re.search(row.cleaned_label.lower(), sentence)      \n        b = re.search(row.dataset_label.lower(), sentence)\n        c = re.search(row.dataset_title.lower(), sentence)\n        cleaned_label = row.cleaned_label.lower()\n        dataset_label = row.dataset_label.lower()\n        dataset_title = row.dataset_title.lower()\n        \n        if  a != None:\n            DATA.append((sentence, cleaned_label))\n            label_count = label_count + 1\n            balanced = True\n        elif b != None:\n            DATA.append((sentence, dataset_label))\n            label_count = label_count + 1\n            balanced = True\n        elif c != None:\n            DATA.append((sentence, dataset_title))\n            label_count = label_count + 1\n            balanced = True            \n        else:\n            if balanced:\n                empty_count = empty_count + 1\n                balanced = False\n    \nprint('Text with dataset:', label_count)\nprint('Text without dataset:', empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:35:22.469125Z","iopub.execute_input":"2021-06-21T01:35:22.469440Z","iopub.status.idle":"2021-06-21T01:57:18.807972Z","shell.execute_reply.started":"2021-06-21T01:35:22.469411Z","shell.execute_reply":"2021-06-21T01:57:18.806995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\ntrain_df = pd.DataFrame(DATA)\ntrain_df = train_df.rename({0: 'Sentence', 1: 'Label'}, axis = 1)\n\n#review\ntrain_df.tail(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.809653Z","iopub.execute_input":"2021-06-21T01:57:18.809961Z","iopub.status.idle":"2021-06-21T01:57:18.864450Z","shell.execute_reply.started":"2021-06-21T01:57:18.809927Z","shell.execute_reply":"2021-06-21T01:57:18.863401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][12345])\nprint('\\n')\nprint(train_df['Label'][12345])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.866055Z","iopub.execute_input":"2021-06-21T01:57:18.866521Z","iopub.status.idle":"2021-06-21T01:57:18.872370Z","shell.execute_reply.started":"2021-06-21T01:57:18.866467Z","shell.execute_reply":"2021-06-21T01:57:18.871658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][45678])\nprint('\\n')\nprint(train_df['Label'][45678])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.873235Z","iopub.execute_input":"2021-06-21T01:57:18.873500Z","iopub.status.idle":"2021-06-21T01:57:18.886958Z","shell.execute_reply.started":"2021-06-21T01:57:18.873473Z","shell.execute_reply":"2021-06-21T01:57:18.885741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][32100])\nprint('\\n')\nprint(train_df['Label'][32100])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.890142Z","iopub.execute_input":"2021-06-21T01:57:18.890548Z","iopub.status.idle":"2021-06-21T01:57:18.902697Z","shell.execute_reply.started":"2021-06-21T01:57:18.890499Z","shell.execute_reply":"2021-06-21T01:57:18.901862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"### Train-Test-Split","metadata":{}},{"cell_type":"code","source":"X = train_df['Sentence'].to_numpy()\ny = train_df['Label'].to_numpy()\n\n#split traing data into training a validation sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 123)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.903635Z","iopub.execute_input":"2021-06-21T01:57:18.903912Z","iopub.status.idle":"2021-06-21T01:57:18.927675Z","shell.execute_reply.started":"2021-06-21T01:57:18.903885Z","shell.execute_reply":"2021-06-21T01:57:18.926309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check shape\nprint('Train sentences:', X_train.shape, '\\n', \n      'Test sentences:', X_test.shape, '\\n', \n      'Train labels:', y_train.shape, '\\n', \n      'Test labels:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.929021Z","iopub.execute_input":"2021-06-21T01:57:18.929376Z","iopub.status.idle":"2021-06-21T01:57:18.941516Z","shell.execute_reply.started":"2021-06-21T01:57:18.929341Z","shell.execute_reply":"2021-06-21T01:57:18.940482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenize","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#limit on the number of features. We use the top 20K features\ntop_k = 20000\n\n#limit on the length of text sequences. Sequences longer than this will be truncated\nmax_sequence_length = 500\n\n#get max sequence length\nmax_length = len(max(X_train, key = len))\nif max_length > max_sequence_length:\n    max_length = max_sequence_length\n    \nmax_vocab_length = 20000 # max number of words to have in our vocabulary\n\n#method to count the unique words in vocabulary and assign each of those words to indices\ntokenizer = Tokenizer(num_words = top_k)\n\n#create vocabulary with training texts\ntokenizer.fit_on_texts(list(X_train))\n\n#convert text into integer sequences\nX_train_seq = tokenizer.texts_to_sequences(X_train)\nX_test_seq = tokenizer.texts_to_sequences(X_test)\n         \n#fix sequence length to max value. \n#sequences shorter than the length are padded in the beginning and sequences longer are truncated at the beginning\n#this turns our lists of integers into a 2D integer tensor of shape (samples, maxlen)\nX_train_pad  = pad_sequences(X_train_seq, maxlen = max_length)\nX_test_pad = pad_sequences(X_test_seq, maxlen = max_length)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:18.942733Z","iopub.execute_input":"2021-06-21T01:57:18.943107Z","iopub.status.idle":"2021-06-21T01:57:23.945513Z","shell.execute_reply.started":"2021-06-21T01:57:18.943070Z","shell.execute_reply":"2021-06-21T01:57:23.944274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of unique words in the training data\nsize_of_vocabulary = len(tokenizer.word_index) + 1 #+1 for padding\nprint(size_of_vocabulary)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:23.946777Z","iopub.execute_input":"2021-06-21T01:57:23.947052Z","iopub.status.idle":"2021-06-21T01:57:23.952115Z","shell.execute_reply.started":"2021-06-21T01:57:23.947024Z","shell.execute_reply":"2021-06-21T01:57:23.950923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:23.953420Z","iopub.execute_input":"2021-06-21T01:57:23.953786Z","iopub.status.idle":"2021-06-21T01:57:23.969028Z","shell.execute_reply.started":"2021-06-21T01:57:23.953743Z","shell.execute_reply":"2021-06-21T01:57:23.967910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encode Label","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#use the LabelEncoder to convert text labels to integers, 0, 1, 2, etc.\nencoder = preprocessing.LabelEncoder()\n\n#since we have two different data set (X_train and X_test), \n#we need to fit it on all of our data otherwise there might be some categories in the test set X_test that were not in the train set X_train \n#and we will get errors\nencoder.fit(list(y_train) + list(y_test)) \ny_train = encoder.transform(y_train)\ny_test = encoder.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:23.970738Z","iopub.execute_input":"2021-06-21T01:57:23.971172Z","iopub.status.idle":"2021-06-21T01:57:24.188501Z","shell.execute_reply.started":"2021-06-21T01:57:23.971124Z","shell.execute_reply":"2021-06-21T01:57:24.187534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train.shape)\nprint('X_test shape:', X_test.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:24.189828Z","iopub.execute_input":"2021-06-21T01:57:24.190181Z","iopub.status.idle":"2021-06-21T01:57:24.196648Z","shell.execute_reply.started":"2021-06-21T01:57:24.190150Z","shell.execute_reply":"2021-06-21T01:57:24.195454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Binarize Label","metadata":{}},{"cell_type":"code","source":"num_classes = train_df['Label'].nunique() + 1\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:24.202768Z","iopub.execute_input":"2021-06-21T01:57:24.203269Z","iopub.status.idle":"2021-06-21T01:57:24.231986Z","shell.execute_reply.started":"2021-06-21T01:57:24.203219Z","shell.execute_reply":"2021-06-21T01:57:24.230922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import utils\n\n#binarize the labels for the neural net\ny_train = utils.to_categorical(y_train, num_classes)\ny_test = utils.to_categorical(y_test, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:24.234264Z","iopub.execute_input":"2021-06-21T01:57:24.234557Z","iopub.status.idle":"2021-06-21T01:57:24.252746Z","shell.execute_reply.started":"2021-06-21T01:57:24.234529Z","shell.execute_reply":"2021-06-21T01:57:24.251969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:24.253807Z","iopub.execute_input":"2021-06-21T01:57:24.254219Z","iopub.status.idle":"2021-06-21T01:57:24.260934Z","shell.execute_reply.started":"2021-06-21T01:57:24.254176Z","shell.execute_reply":"2021-06-21T01:57:24.259863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Pretrained Word Vector","metadata":{}},{"cell_type":"code","source":"#load the whole embedding into memory\nembeddings_index = {}\nf = open('../input/glove840b300dtxt/glove.840B.300d.txt')\n\nfor line in tqdm(f):\n    values = line.split()\n    word = values[0]\n    try:\n        coefs = np.asarray(values[1:], dtype = 'float32')\n        embeddings_index[word] = coefs\n    except ValueError: #catch the exception where there are strings in the GloVe text file, can be avoided if use glove.42B.300d.txt\n        pass\n    \nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T01:57:24.262306Z","iopub.execute_input":"2021-06-21T01:57:24.262589Z","iopub.status.idle":"2021-06-21T02:02:04.368626Z","shell.execute_reply.started":"2021-06-21T01:57:24.262561Z","shell.execute_reply":"2021-06-21T02:02:04.367104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:04.371193Z","iopub.execute_input":"2021-06-21T02:02:04.371753Z","iopub.status.idle":"2021-06-21T02:02:04.695448Z","shell.execute_reply.started":"2021-06-21T02:02:04.371692Z","shell.execute_reply":"2021-06-21T02:02:04.694417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELING","metadata":{}},{"cell_type":"code","source":"#simple bidirectional LSTM with GloVe embeddings and Dense layers\nLSTM_model = Sequential()\n\n#embedding layer\nLSTM_model.add(Embedding(size_of_vocabulary, 300, \n                          weights = [embedding_matrix], #load GloVe\n                          input_length = X_train_pad.shape[0], \n                          trainable = False)) #keep frozen\n\n#lstm layer\nLSTM_model.add(LSTM(128, return_sequences = True, dropout = 0.2))\nLSTM_model.add(SpatialDropout1D(0.3))\nLSTM_model.add(Bidirectional(LSTM(300, dropout = 0.3, recurrent_dropout = 0.3)))\n    \n#fully connected layers\nLSTM_model.add(Dense(512, activation = 'relu'))\nLSTM_model.add(Dropout(0.3))\n\nLSTM_model.add(Dense(128, activation = 'relu')) \nLSTM_model.add(Dropout(0.3))\n\nLSTM_model.add(Dense(64, activation = 'relu')) \nLSTM_model.add(Dropout(0.3))\n                          \n#output layer\nLSTM_model.add(Dense(num_classes, activation = 'softmax')) ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:04.696853Z","iopub.execute_input":"2021-06-21T02:02:04.697384Z","iopub.status.idle":"2021-06-21T02:02:06.396757Z","shell.execute_reply.started":"2021-06-21T02:02:04.697337Z","shell.execute_reply":"2021-06-21T02:02:06.395645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:06.401022Z","iopub.execute_input":"2021-06-21T02:02:06.401393Z","iopub.status.idle":"2021-06-21T02:02:06.408957Z","shell.execute_reply.started":"2021-06-21T02:02:06.401359Z","shell.execute_reply":"2021-06-21T02:02:06.407595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary\nLSTM_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:06.410415Z","iopub.execute_input":"2021-06-21T02:02:06.410721Z","iopub.status.idle":"2021-06-21T02:02:06.431560Z","shell.execute_reply.started":"2021-06-21T02:02:06.410688Z","shell.execute_reply":"2021-06-21T02:02:06.430365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nplot_model(LSTM_model, to_file = 'lstm_model_plot.png', show_shapes = True, show_layer_names = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:06.433191Z","iopub.execute_input":"2021-06-21T02:02:06.433573Z","iopub.status.idle":"2021-06-21T02:02:07.252222Z","shell.execute_reply.started":"2021-06-21T02:02:06.433539Z","shell.execute_reply":"2021-06-21T02:02:07.251342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compile\nLSTM_model.compile(optimizer = keras.optimizers.Adam(0.0001), #low learning rate is good, but the model will take more iterations to converge\n                    loss = 'categorical_crossentropy',\n                    metrics = ['acc'])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:07.253686Z","iopub.execute_input":"2021-06-21T02:02:07.254133Z","iopub.status.idle":"2021-06-21T02:02:07.274552Z","shell.execute_reply.started":"2021-06-21T02:02:07.254096Z","shell.execute_reply":"2021-06-21T02:02:07.273520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.utils import class_weight\n\n#correct class imbalance\n#class_weights = list(class_weight.compute_class_weight('balanced',\n#                                                       np.unique(train_df['Label']),\n#                                                       train_df['Label']))\n\n#weights = {}\n#for index, weight in enumerate(class_weights) : weights[index] = weight","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:07.275873Z","iopub.execute_input":"2021-06-21T02:02:07.276390Z","iopub.status.idle":"2021-06-21T02:02:07.282932Z","shell.execute_reply.started":"2021-06-21T02:02:07.276340Z","shell.execute_reply":"2021-06-21T02:02:07.281811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.datetime.now()\nbatch_size = 256\n\n#fit\nLSTM_history = LSTM_model.fit(np.array(X_train_pad), np.array(y_train),\n                                  #class_weight = weights,\n                                  batch_size = batch_size,\n                                  epochs = 30,\n                                  validation_data = (np.array(X_test_pad), np.array(y_test)),\n                                  steps_per_epoch = X_train_pad.shape[0] // 256,\n                                  validation_steps = X_test_pad.shape[0] // 256,\n                                  callbacks = early_stopping)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T02:02:07.284699Z","iopub.execute_input":"2021-06-21T02:02:07.285335Z","iopub.status.idle":"2021-06-21T03:02:07.412504Z","shell.execute_reply.started":"2021-06-21T02:02:07.285290Z","shell.execute_reply":"2021-06-21T03:02:07.410351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = datetime.datetime.now()\nelapsed = end - start\nprint('Training took a total of {}'.format(elapsed))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.413830Z","iopub.status.idle":"2021-06-21T03:02:07.414316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model\nLSTM_model.save('lstm_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.415324Z","iopub.status.idle":"2021-06-21T03:02:07.415720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig , ax = plt.subplots(1,2)\nfig.set_size_inches(20, 8)\n\nLSTM_train_acc = LSTM_history.history['acc']\nLSTM_train_loss = LSTM_history.history['loss']\nLSTM_val_acc = LSTM_history.history['val_acc']\nLSTM_val_loss = LSTM_history.history['val_loss']\n\nepochs = range(1, len(LSTM_train_acc) + 1)\n\nax[0].plot(epochs, LSTM_train_acc , 'go-' , label = 'Training Accuracy')\nax[0].plot(epochs , LSTM_val_acc , 'yo-' , label = 'Validation Accuracy')\nax[0].set_title('LSTM Model Train & Validation Accuracy')\nax[0].legend()\nax[0].set_xlabel('Epochs')\nax[0].set_ylabel('Accuracy')\n\nax[1].plot(epochs, LSTM_train_loss , 'go-' , label = 'Training Loss')\nax[1].plot(epochs, LSTM_val_loss , 'yo-' , label = 'Validation Loss')\nax[1].set_title('LSTM Model Train & Validation Loss')\nax[1].legend()\nax[1].set_xlabel('Epochs')\nax[1].set_ylabel('Loss')\n\nplt.show()\n\n#save\nplt.savefig('lstm_acc_loss.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.416631Z","iopub.status.idle":"2021-06-21T03:02:07.417066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nplt.savefig('lstm_acc_loss.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.418041Z","iopub.status.idle":"2021-06-21T03:02:07.418483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train loss & accuracy:', LSTM_model.evaluate(X_train_pad, y_train))\nprint('\\n')\nprint('Test loss & accuracy:', LSTM_model.evaluate(X_test_pad, y_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.419363Z","iopub.status.idle":"2021-06-21T03:02:07.419779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make prediction\nLSTM_yhat_test = LSTM_model.predict(X_test_pad)\n\n#to evaluate accuracy we need a vector of labels\nLSTM_yhat_test = np.argmax(LSTM_yhat_test, axis = 1)\nLSTM_y_test = np.argmax(y_test, axis = 1)\n\n#get classification report\nprint('Model: LSTM', '\\n', classification_report(LSTM_y_test, LSTM_yhat_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.420667Z","iopub.status.idle":"2021-06-21T03:02:07.421099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary table\nsummary_table = pd.DataFrame({'Model': [],\n                              'Accuracy': [],\n                              'Precision': [], 'Recall': [], 'F1': []})","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.421933Z","iopub.status.idle":"2021-06-21T03:02:07.422412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#update summary table\nsummary_table.loc[1] = ['RNN Bidirectional LSTM',\n                        round(accuracy_score(LSTM_y_test, LSTM_yhat_test), 2),\n                        round(precision_score(LSTM_y_test, LSTM_yhat_test, average = 'macro'), 2), \n                        round(recall_score(LSTM_y_test, LSTM_yhat_test, average = 'macro'), 2), \n                        round(f1_score(LSTM_y_test, LSTM_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.423471Z","iopub.status.idle":"2021-06-21T03:02:07.423919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table.to_csv('lstm_summary_table.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.424827Z","iopub.status.idle":"2021-06-21T03:02:07.425304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"#get text\ntqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_json_pub)\n\n#clean text\ntrain['text'] = train['text'].progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.426698Z","iopub.status.idle":"2021-06-21T03:02:07.427153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\n#read data\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(read_json_pub, train_path = test_path))\n\n#review\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.428211Z","iopub.status.idle":"2021-06-21T03:02:07.428663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.429813Z","iopub.status.idle":"2021-06-21T03:02:07.430241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_matching = True\nlstm_prediction = True\n\nid_list = []\nlabels_list = []\n\nfor index, row in tqdm(sample_submission.iterrows()):\n\n    sample_text = row['text']\n\n    row_id = row['Id']\n    \n    #takes only the rows where train file is identical to a test file\n    temp_df = train[train['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    #literal_matching \n    if literal_matching:\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():    \n                cleaned_labels.append(clean_text(known_label))\n            \n        print('cleaned label:', set(cleaned_labels))   \n    \n    #lstm_prediction \n    if lstm_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n        \n        tokenizer.fit_on_texts([sentence])\n        sentence_seq = tokenizer.texts_to_sequences([sentence])\n        sentence_pad  = pad_sequences(sentence_seq, maxlen = max_length)\n            \n        #predict\n        lstm_labels = LSTM_model.predict(sentence_pad)\n    \n        #get label\n        lstm_labels = encoder.inverse_transform([np.argmax(lstm_labels)])\n        print('lstm label:', set(lstm_labels))\n        lstm_labels = set(lstm_labels)\n        \n    cleaned_labels += lstm_labels\n    print('updated cleaned label:', set(cleaned_labels))\n        \n    cleaned_labels = set(cleaned_labels)\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]    \n    labels_list.append('|'.join(cleaned_labels))\n    print('label list:', labels_list)   \n    id_list.append(row_id)\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.431230Z","iopub.status.idle":"2021-06-21T03:02:07.431667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\nsample_submission['PredictionString'] = labels_list\nsample_submission.drop(columns = 'text', axis = 1, inplace = True)\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.432519Z","iopub.status.idle":"2021-06-21T03:02:07.432942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission['PredictionString'][0])\nprint('\\n')\nprint(sample_submission['PredictionString'][1])\nprint('\\n')\nprint(sample_submission['PredictionString'][2])\nprint('\\n')\nprint(sample_submission['PredictionString'][3])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.434095Z","iopub.status.idle":"2021-06-21T03:02:07.434531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsample_submission.to_csv('submission.csv', index = False)\n\n#check\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-21T03:02:07.435527Z","iopub.status.idle":"2021-06-21T03:02:07.435943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}