{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T06:00:10.772009Z","iopub.execute_input":"2021-06-20T06:00:10.772458Z","iopub.status.idle":"2021-06-20T06:00:23.397244Z","shell.execute_reply.started":"2021-06-20T06:00:10.772359Z","shell.execute_reply":"2021-06-20T06:00:23.395994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\n\nimport spacy\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:23.398581Z","iopub.execute_input":"2021-06-20T06:00:23.398865Z","iopub.status.idle":"2021-06-20T06:00:32.380810Z","shell.execute_reply.started":"2021-06-20T06:00:23.398837Z","shell.execute_reply":"2021-06-20T06:00:32.379786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\n\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.382611Z","iopub.execute_input":"2021-06-20T06:00:32.382893Z","iopub.status.idle":"2021-06-20T06:00:32.401657Z","shell.execute_reply.started":"2021-06-20T06:00:32.382865Z","shell.execute_reply":"2021-06-20T06:00:32.400705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.403264Z","iopub.execute_input":"2021-06-20T06:00:32.403585Z","iopub.status.idle":"2021-06-20T06:00:32.408198Z","shell.execute_reply.started":"2021-06-20T06:00:32.403556Z","shell.execute_reply":"2021-06-20T06:00:32.407113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://towardsdatascience.com/text-analysis-feature-engineering-with-nlp-502d6ea9225d\n\ndef text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.409520Z","iopub.execute_input":"2021-06-20T06:00:32.409812Z","iopub.status.idle":"2021-06-20T06:00:32.420381Z","shell.execute_reply.started":"2021-06-20T06:00:32.409784Z","shell.execute_reply":"2021-06-20T06:00:32.419223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n    \ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.423392Z","iopub.execute_input":"2021-06-20T06:00:32.424556Z","iopub.status.idle":"2021-06-20T06:00:32.432803Z","shell.execute_reply.started":"2021-06-20T06:00:32.424519Z","shell.execute_reply":"2021-06-20T06:00:32.431683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.434154Z","iopub.execute_input":"2021-06-20T06:00:32.434782Z","iopub.status.idle":"2021-06-20T06:00:32.448235Z","shell.execute_reply.started":"2021-06-20T06:00:32.434748Z","shell.execute_reply":"2021-06-20T06:00:32.447430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.451060Z","iopub.execute_input":"2021-06-20T06:00:32.451697Z","iopub.status.idle":"2021-06-20T06:00:32.460152Z","shell.execute_reply.started":"2021-06-20T06:00:32.451664Z","shell.execute_reply":"2021-06-20T06:00:32.459189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read \ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\n#review\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.462169Z","iopub.execute_input":"2021-06-20T06:00:32.462500Z","iopub.status.idle":"2021-06-20T06:00:32.633238Z","shell.execute_reply.started":"2021-06-20T06:00:32.462470Z","shell.execute_reply":"2021-06-20T06:00:32.632363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Sentences & Labels","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/jagdmir/spacy-ner-model\n\nimport nltk\n\nDATA = []\nlabel_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n    \n    balanced = False\n\n    sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(data))]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    for sentence in sentences:          \n      \n        a = re.search(row.cleaned_label.lower(), sentence)\n        b = re.search(row.dataset_label.lower(), sentence)\n        c = re.search(row.dataset_title.lower(), sentence)\n        cleaned_label = row.cleaned_label.lower()\n        dataset_label = row.dataset_label.lower()\n        dataset_title = row.dataset_title.lower()\n        \n        if  a != None:\n            DATA.append((sentence, cleaned_label))\n            label_count = label_count + 1\n            balanced = True\n        elif b != None:\n            DATA.append((sentence, dataset_label))\n            label_count = label_count + 1\n            balanced = True\n        elif c != None:\n            DATA.append((sentence, dataset_title))\n            label_count = label_count + 1\n            balanced = True  \n            \n        else:\n            pass\n                \nprint(\"Text with dataset:\", label_count)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:00:32.634455Z","iopub.execute_input":"2021-06-20T06:00:32.634734Z","iopub.status.idle":"2021-06-20T06:22:15.188626Z","shell.execute_reply.started":"2021-06-20T06:00:32.634707Z","shell.execute_reply":"2021-06-20T06:22:15.187558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\ntrain_df = pd.DataFrame(DATA)\ntrain_df = train_df.rename({0: 'Sentence', 1: 'Label'}, axis = 1)\n\n#review\ntrain_df.tail(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.190156Z","iopub.execute_input":"2021-06-20T06:22:15.190585Z","iopub.status.idle":"2021-06-20T06:22:15.245454Z","shell.execute_reply.started":"2021-06-20T06:22:15.190547Z","shell.execute_reply":"2021-06-20T06:22:15.244470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Sentence:', train_df['Sentence'][555])\nprint('\\n')\nprint('Label:', train_df['Label'][555])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.246789Z","iopub.execute_input":"2021-06-20T06:22:15.247171Z","iopub.status.idle":"2021-06-20T06:22:15.253673Z","shell.execute_reply.started":"2021-06-20T06:22:15.247137Z","shell.execute_reply":"2021-06-20T06:22:15.252628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Sentence:', train_df['Sentence'][50000])\nprint('\\n')\nprint('Label:', train_df['Label'][50000])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.255136Z","iopub.execute_input":"2021-06-20T06:22:15.255765Z","iopub.status.idle":"2021-06-20T06:22:15.270305Z","shell.execute_reply.started":"2021-06-20T06:22:15.255716Z","shell.execute_reply":"2021-06-20T06:22:15.268990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"### Train-Test-Split","metadata":{}},{"cell_type":"code","source":"X = train_df['Sentence'].to_numpy()\ny = train_df['Label'].to_numpy()\n\n#split traing data into training a validation sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 123)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.271737Z","iopub.execute_input":"2021-06-20T06:22:15.272127Z","iopub.status.idle":"2021-06-20T06:22:15.290845Z","shell.execute_reply.started":"2021-06-20T06:22:15.272095Z","shell.execute_reply":"2021-06-20T06:22:15.289986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check shape\nprint('Train sentences:', X_train.shape, '\\n', \n      'Test sentences:', X_test.shape, '\\n', \n      'Train labels:', y_train.shape, '\\n', \n      'Test labels:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.292026Z","iopub.execute_input":"2021-06-20T06:22:15.292501Z","iopub.status.idle":"2021-06-20T06:22:15.312606Z","shell.execute_reply.started":"2021-06-20T06:22:15.292456Z","shell.execute_reply":"2021-06-20T06:22:15.311504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenize","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#limit on the number of features. We use the top 20K features\ntop_k = 20000\n\n#limit on the length of text sequences. Sequences longer than this will be truncated\nmax_sequence_length = 500\n\n#get max sequence length\nmax_length = len(max(X_train, key = len))\nif max_length > max_sequence_length:\n    max_length = max_sequence_length\n    \nmax_vocab_length = 20000 # max number of words to have in our vocabulary\n\n#method to count the unique words in vocabulary and assign each of those words to indices\ntokenizer = Tokenizer(num_words = top_k)\n\n#create vocabulary with training texts\ntokenizer.fit_on_texts(list(X_train))\n\n#convert text into integer sequences\nX_train_seq = tokenizer.texts_to_sequences(X_train)\nX_test_seq = tokenizer.texts_to_sequences(X_test)\n         \n#fix sequence length to max value. \n#sequences shorter than the length are padded in the beginning and sequences longer are truncated at the beginning\n#this turns our lists of integers into a 2D integer tensor of shape (samples, maxlen)\nX_train_pad  = pad_sequences(X_train_seq, maxlen = max_length)\nX_test_pad = pad_sequences(X_test_seq, maxlen = max_length)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:15.313630Z","iopub.execute_input":"2021-06-20T06:22:15.313999Z","iopub.status.idle":"2021-06-20T06:22:20.448927Z","shell.execute_reply.started":"2021-06-20T06:22:15.313969Z","shell.execute_reply":"2021-06-20T06:22:20.447929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of unique words in the training data\nsize_of_vocabulary = len(tokenizer.word_index) + 1 #+1 for padding\nprint(size_of_vocabulary)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.450116Z","iopub.execute_input":"2021-06-20T06:22:20.450407Z","iopub.status.idle":"2021-06-20T06:22:20.456547Z","shell.execute_reply.started":"2021-06-20T06:22:20.450378Z","shell.execute_reply":"2021-06-20T06:22:20.454876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.458020Z","iopub.execute_input":"2021-06-20T06:22:20.458329Z","iopub.status.idle":"2021-06-20T06:22:20.475340Z","shell.execute_reply.started":"2021-06-20T06:22:20.458285Z","shell.execute_reply":"2021-06-20T06:22:20.473957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encode Label","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#use the LabelEncoder to convert text labels to integers, 0, 1, 2, etc.\nencoder = preprocessing.LabelEncoder()\n\n#since we have two different data set (X_train and X_test), \n#we need to fit it on all of our data otherwise there might be some categories in the test set X_test that were not in the train set X_train \n#and we will get errors\nencoder.fit(list(y_train) + list(y_test)) \ny_train = encoder.transform(y_train)\ny_test = encoder.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.476428Z","iopub.execute_input":"2021-06-20T06:22:20.476767Z","iopub.status.idle":"2021-06-20T06:22:20.697457Z","shell.execute_reply.started":"2021-06-20T06:22:20.476736Z","shell.execute_reply":"2021-06-20T06:22:20.696474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train.shape)\nprint('X_test shape:', X_test.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.698916Z","iopub.execute_input":"2021-06-20T06:22:20.699204Z","iopub.status.idle":"2021-06-20T06:22:20.706917Z","shell.execute_reply.started":"2021-06-20T06:22:20.699175Z","shell.execute_reply":"2021-06-20T06:22:20.705562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder.classes_","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.708205Z","iopub.execute_input":"2021-06-20T06:22:20.708571Z","iopub.status.idle":"2021-06-20T06:22:20.721255Z","shell.execute_reply.started":"2021-06-20T06:22:20.708539Z","shell.execute_reply":"2021-06-20T06:22:20.720214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Binarize Label","metadata":{}},{"cell_type":"code","source":"num_classes = train_df['Label'].nunique() + 1\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.722630Z","iopub.execute_input":"2021-06-20T06:22:20.722909Z","iopub.status.idle":"2021-06-20T06:22:20.754069Z","shell.execute_reply.started":"2021-06-20T06:22:20.722882Z","shell.execute_reply":"2021-06-20T06:22:20.752852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import utils\n\n#binarize the labels for the neural net\ny_train = utils.to_categorical(y_train, num_classes)\ny_test = utils.to_categorical(y_test, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.758934Z","iopub.execute_input":"2021-06-20T06:22:20.759276Z","iopub.status.idle":"2021-06-20T06:22:20.780721Z","shell.execute_reply.started":"2021-06-20T06:22:20.759246Z","shell.execute_reply":"2021-06-20T06:22:20.779641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.783025Z","iopub.execute_input":"2021-06-20T06:22:20.783335Z","iopub.status.idle":"2021-06-20T06:22:20.789975Z","shell.execute_reply.started":"2021-06-20T06:22:20.783293Z","shell.execute_reply":"2021-06-20T06:22:20.788818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Pretrained Word Vector","metadata":{}},{"cell_type":"code","source":"#load the whole embedding into memory\nembeddings_index = {}\nf = open('../input/glove840b300dtxt/glove.840B.300d.txt')\n\nfor line in tqdm(f):\n    values = line.split()\n    word = values[0]\n    try:\n        coefs = np.asarray(values[1:], dtype = 'float32')\n        embeddings_index[word] = coefs\n    except ValueError: #catch the exception where there are strings in the GloVe text file, can be avoided if use glove.42B.300d.txt\n        pass\n    \nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:22:20.791349Z","iopub.execute_input":"2021-06-20T06:22:20.791675Z","iopub.status.idle":"2021-06-20T06:26:48.374376Z","shell.execute_reply.started":"2021-06-20T06:22:20.791642Z","shell.execute_reply":"2021-06-20T06:26:48.372801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:48.377102Z","iopub.execute_input":"2021-06-20T06:26:48.377542Z","iopub.status.idle":"2021-06-20T06:26:48.807418Z","shell.execute_reply.started":"2021-06-20T06:26:48.377488Z","shell.execute_reply":"2021-06-20T06:26:48.806416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELING","metadata":{}},{"cell_type":"code","source":"#hyperparameters\nfilters = 4\nkernel_size = 3\ndropout_rate = 0.5\npool_size = 3","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:48.809157Z","iopub.execute_input":"2021-06-20T06:26:48.809611Z","iopub.status.idle":"2021-06-20T06:26:48.816942Z","shell.execute_reply.started":"2021-06-20T06:26:48.809566Z","shell.execute_reply":"2021-06-20T06:26:48.814456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#simple bidirectional LSTM with GloVe embeddings and Dense layers\nsepcnn_model = Sequential()\n\n#embedding layer\nsepcnn_model.add(Embedding(size_of_vocabulary, \n                           300,\n                           weights = [embedding_matrix], #load GloVe\n                           input_length = X_train_pad.shape[0],\n                           trainable = False)) #keep frozen\n\nsepcnn_model.add(Dropout(rate = dropout_rate))\nsepcnn_model.add(SeparableConv1D(filters = filters,\n                                  kernel_size = kernel_size,\n                                  activation = 'relu'))\nsepcnn_model.add(SeparableConv1D(filters = filters,\n                                  kernel_size = kernel_size,\n                                  activation = 'relu'))\n    \nsepcnn_model.add(MaxPooling1D(pool_size = pool_size))\nsepcnn_model.add(SeparableConv1D(filters = filters * 2,\n                              kernel_size = kernel_size,\n                              activation = 'relu'))\n\nsepcnn_model.add(SeparableConv1D(filters = filters * 2,\n                              kernel_size = kernel_size,\n                              activation = 'relu'))\n                 \nsepcnn_model.add(GlobalAveragePooling1D())\n\nsepcnn_model.add(Dropout(rate = dropout_rate))\n                          \n#output layer\nsepcnn_model.add(Dense(num_classes, activation = 'softmax')) ","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:48.818629Z","iopub.execute_input":"2021-06-20T06:26:48.818924Z","iopub.status.idle":"2021-06-20T06:26:49.261952Z","shell.execute_reply.started":"2021-06-20T06:26:48.818894Z","shell.execute_reply":"2021-06-20T06:26:49.260971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary\nsepcnn_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:49.263447Z","iopub.execute_input":"2021-06-20T06:26:49.263755Z","iopub.status.idle":"2021-06-20T06:26:49.277016Z","shell.execute_reply.started":"2021-06-20T06:26:49.263725Z","shell.execute_reply":"2021-06-20T06:26:49.275742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nplot_model(sepcnn_model, to_file = 'sepcnn_model_plot.png', show_shapes = True, show_layer_names = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:49.278892Z","iopub.execute_input":"2021-06-20T06:26:49.279276Z","iopub.status.idle":"2021-06-20T06:26:49.990794Z","shell.execute_reply.started":"2021-06-20T06:26:49.279215Z","shell.execute_reply":"2021-06-20T06:26:49.989568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compile\nsepcnn_model.compile(optimizer = keras.optimizers.Adam(0.0001), #low learning rate is good, but the model will take more iterations to converge\n                    loss = 'categorical_crossentropy',\n                    metrics = ['acc'])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:49.992525Z","iopub.execute_input":"2021-06-20T06:26:49.992875Z","iopub.status.idle":"2021-06-20T06:26:50.012761Z","shell.execute_reply.started":"2021-06-20T06:26:49.992839Z","shell.execute_reply":"2021-06-20T06:26:50.011813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.utils import class_weight\n\n#correct class imbalance\n#class_weights = list(class_weight.compute_class_weight('balanced',\n#                                                       np.unique(train_df['Label']),\n#                                                       train_df['Label']))\n\n#weights = {}\n#for index, weight in enumerate(class_weights) : weights[index] = weight","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:50.014249Z","iopub.execute_input":"2021-06-20T06:26:50.014591Z","iopub.status.idle":"2021-06-20T06:26:50.025822Z","shell.execute_reply.started":"2021-06-20T06:26:50.014561Z","shell.execute_reply":"2021-06-20T06:26:50.024622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:', X_train_pad.shape)\nprint('X_test shape:', X_test_pad.shape)\nprint('y_train shape:', y_train.shape)\nprint('y_test shape:', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:50.027670Z","iopub.execute_input":"2021-06-20T06:26:50.028022Z","iopub.status.idle":"2021-06-20T06:26:50.043601Z","shell.execute_reply.started":"2021-06-20T06:26:50.027991Z","shell.execute_reply":"2021-06-20T06:26:50.042844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add callbacks\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\n\n#define the callbacks\nearly_stopping = [EarlyStopping(monitor = 'val_loss', patience = 5, verbose = 1),\n                 ModelCheckpoint(filepath = 'sepcnn_model.h5', monitor = 'val_loss', save_best_only = True)]","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:50.045145Z","iopub.execute_input":"2021-06-20T06:26:50.045689Z","iopub.status.idle":"2021-06-20T06:26:50.059382Z","shell.execute_reply.started":"2021-06-20T06:26:50.045656Z","shell.execute_reply":"2021-06-20T06:26:50.058017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = datetime.datetime.now()\nbatch_size = 256\n\n#fit\nsepcnn_history = sepcnn_model.fit(np.array(X_train_pad), np.array(y_train),\n                                  #class_weight = weights,\n                                  batch_size = batch_size,\n                                  epochs = 20,\n                                  validation_data = (np.array(X_test_pad), np.array(y_test)),\n                                  steps_per_epoch = X_train_pad.shape[0] // 256,\n                                  validation_steps = X_test_pad.shape[0] // 256,\n                                  callbacks = early_stopping)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:26:50.061118Z","iopub.execute_input":"2021-06-20T06:26:50.061578Z","iopub.status.idle":"2021-06-20T07:11:54.685720Z","shell.execute_reply.started":"2021-06-20T06:26:50.061539Z","shell.execute_reply":"2021-06-20T07:11:54.684642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = datetime.datetime.now()\nelapsed = end - start\nprint('Training took a total of {}'.format(elapsed))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:11:54.687704Z","iopub.execute_input":"2021-06-20T07:11:54.688140Z","iopub.status.idle":"2021-06-20T07:11:54.693237Z","shell.execute_reply.started":"2021-06-20T07:11:54.688101Z","shell.execute_reply":"2021-06-20T07:11:54.692409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model\nsepcnn_model.save('sepcnn_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:11:54.694696Z","iopub.execute_input":"2021-06-20T07:11:54.695012Z","iopub.status.idle":"2021-06-20T07:11:54.840666Z","shell.execute_reply.started":"2021-06-20T07:11:54.694982Z","shell.execute_reply":"2021-06-20T07:11:54.839702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig , ax = plt.subplots(1,2)\nfig.set_size_inches(20, 8)\n\nsepcnn_train_acc = sepcnn_history.history['acc']\nsepcnn_train_loss = sepcnn_history.history['loss']\nsepcnn_val_acc = sepcnn_history.history['val_acc']\nsepcnn_val_loss = sepcnn_history.history['val_loss']\n\nepochs = range(1, len(sepcnn_train_acc) + 1)\n\nax[0].plot(epochs , sepcnn_train_acc , 'g-o' , label = 'Training Accuracy')\nax[0].plot(epochs , sepcnn_val_acc , 'y-o' , label = 'Validation Accuracy')\nax[0].set_title('SEPCNN Model Train & Validation Accuracy')\nax[0].legend(loc = 'lower right')\nax[0].set_xlabel('Epochs')\nax[0].set_ylabel('Accuracy')\n\nax[1].plot(epochs, sepcnn_train_loss , 'g-o' , label = 'Training Loss')\nax[1].plot(epochs, sepcnn_val_loss , 'y-o' , label = 'Validation Loss')\nax[1].set_title('SEPCNN Model Train & Validation Loss')\nax[1].legend()\nax[1].set_xlabel('Epochs')\nax[1].set_ylabel('Accuracy')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:11:54.841964Z","iopub.execute_input":"2021-06-20T07:11:54.842458Z","iopub.status.idle":"2021-06-20T07:11:55.270164Z","shell.execute_reply.started":"2021-06-20T07:11:54.842411Z","shell.execute_reply":"2021-06-20T07:11:55.269113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nplt.savefig('sepcnn_acc_loss.png')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:11:55.271781Z","iopub.execute_input":"2021-06-20T07:11:55.272090Z","iopub.status.idle":"2021-06-20T07:11:55.289091Z","shell.execute_reply.started":"2021-06-20T07:11:55.272061Z","shell.execute_reply":"2021-06-20T07:11:55.288062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train loss & accuracy:', sepcnn_model.evaluate(X_train_pad, y_train))\nprint('\\n')\nprint('Test loss & accuracy:', sepcnn_model.evaluate(X_test_pad, y_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:11:55.290831Z","iopub.execute_input":"2021-06-20T07:11:55.291264Z","iopub.status.idle":"2021-06-20T07:12:25.800007Z","shell.execute_reply.started":"2021-06-20T07:11:55.291219Z","shell.execute_reply":"2021-06-20T07:12:25.798946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make prediction\nsepcnn_yhat_test = sepcnn_model.predict(X_test_pad)\n\n#to evaluate accuracy we need a vector of labels\nsepcnn_yhat_test = np.argmax(sepcnn_yhat_test, axis = 1)\nsepcnn_y_test = np.argmax(y_test, axis = 1)\n\n#get classification report\nprint('Model: SEPCNN', '\\n', classification_report(sepcnn_y_test, sepcnn_yhat_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:25.801497Z","iopub.execute_input":"2021-06-20T07:12:25.801906Z","iopub.status.idle":"2021-06-20T07:12:31.778873Z","shell.execute_reply.started":"2021-06-20T07:12:25.801875Z","shell.execute_reply":"2021-06-20T07:12:31.777831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = encoder.inverse_transform([np.argmax(sepcnn_yhat_test)]) \npreds","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:31.780177Z","iopub.execute_input":"2021-06-20T07:12:31.780501Z","iopub.status.idle":"2021-06-20T07:12:31.786483Z","shell.execute_reply.started":"2021-06-20T07:12:31.780469Z","shell.execute_reply":"2021-06-20T07:12:31.785487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary table\nsummary_table = pd.DataFrame({'Model': [],\n                              'Accuracy': [],\n                              'Precision': [], 'Recall': [], 'F1': []})","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:31.787662Z","iopub.execute_input":"2021-06-20T07:12:31.787948Z","iopub.status.idle":"2021-06-20T07:12:31.802873Z","shell.execute_reply.started":"2021-06-20T07:12:31.787919Z","shell.execute_reply":"2021-06-20T07:12:31.802080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#update summary table\nsummary_table.loc[0] = ['DL SEPCNN',\n                        round(accuracy_score(sepcnn_y_test, sepcnn_yhat_test), 2),\n                        round(precision_score(sepcnn_y_test, sepcnn_yhat_test, average = 'macro'), 2), \n                        round(recall_score(sepcnn_y_test, sepcnn_yhat_test, average = 'macro'), 2), \n                        round(f1_score(sepcnn_y_test, sepcnn_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:31.804269Z","iopub.execute_input":"2021-06-20T07:12:31.804904Z","iopub.status.idle":"2021-06-20T07:12:31.849152Z","shell.execute_reply.started":"2021-06-20T07:12:31.804857Z","shell.execute_reply":"2021-06-20T07:12:31.848480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table.to_csv('sepcnn_summary_table.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:31.850677Z","iopub.execute_input":"2021-06-20T07:12:31.851259Z","iopub.status.idle":"2021-06-20T07:12:31.861126Z","shell.execute_reply.started":"2021-06-20T07:12:31.851212Z","shell.execute_reply":"2021-06-20T07:12:31.859937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"#get text\ntqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_json_pub)\n\n#clean text\ntrain['text'] = train['text'].progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:12:31.862631Z","iopub.execute_input":"2021-06-20T07:12:31.863034Z","iopub.status.idle":"2021-06-20T07:15:18.157048Z","shell.execute_reply.started":"2021-06-20T07:12:31.862992Z","shell.execute_reply":"2021-06-20T07:15:18.156235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\n#read data\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(read_json_pub, train_path = test_path))\n\n#review\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:18.158462Z","iopub.execute_input":"2021-06-20T07:15:18.158948Z","iopub.status.idle":"2021-06-20T07:15:18.281747Z","shell.execute_reply.started":"2021-06-20T07:15:18.158914Z","shell.execute_reply":"2021-06-20T07:15:18.280517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:18.283140Z","iopub.execute_input":"2021-06-20T07:15:18.283490Z","iopub.status.idle":"2021-06-20T07:15:18.296262Z","shell.execute_reply.started":"2021-06-20T07:15:18.283458Z","shell.execute_reply":"2021-06-20T07:15:18.295406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_matching = True\nsepcnn_prediction = True\n\nid_list = []\nlabels_list = []\n\nfor index, row in tqdm(sample_submission.iterrows()):\n\n    sample_text = row['text']\n\n    row_id = row['Id']\n    \n    #check if the sample text is equal to one of the train samples and if so, use those labels\n    temp_df = train[train['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    #literal_matching for known label in sample text\n    if literal_matching:\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():    \n                cleaned_labels.append(clean_text(known_label))\n            \n        print('cleaned label:', set(cleaned_labels))   \n    \n    #lstm_prediction \n    if sepcnn_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars        \n        \n        tokenizer.fit_on_texts(list(sentence))\n        sentence_seq = tokenizer.texts_to_sequences([sentence])\n        sentence_pad  = pad_sequences(sentence_seq, maxlen = max_length)\n            \n        #predict\n        sepcnn_labels = sepcnn_model.predict(sentence_pad)\n    \n        #get label\n        sepcnn_labels = encoder.inverse_transform([np.argmax(sepcnn_labels)])\n            \n        print('sepcnn label:', set(sepcnn_labels))\n        sepcnn_labels = set(sepcnn_labels)\n        \n    cleaned_labels += sepcnn_labels\n    print('updated cleaned label:', set(cleaned_labels))\n        \n    cleaned_labels = set(cleaned_labels)\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]    \n    labels_list.append('|'.join(cleaned_labels))\n    print('label list:', labels_list)   \n    id_list.append(row_id)\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:18.297721Z","iopub.execute_input":"2021-06-20T07:15:18.298014Z","iopub.status.idle":"2021-06-20T07:15:19.255943Z","shell.execute_reply.started":"2021-06-20T07:15:18.297983Z","shell.execute_reply":"2021-06-20T07:15:19.254822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\nsample_submission['PredictionString'] = labels_list\nsample_submission.drop(columns = 'text', axis = 1, inplace = True)\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:19.257434Z","iopub.execute_input":"2021-06-20T07:15:19.257737Z","iopub.status.idle":"2021-06-20T07:15:19.272424Z","shell.execute_reply.started":"2021-06-20T07:15:19.257702Z","shell.execute_reply":"2021-06-20T07:15:19.271290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission['PredictionString'][0])\nprint('\\n')\nprint(sample_submission['PredictionString'][1])\nprint('\\n')\nprint(sample_submission['PredictionString'][2])\nprint('\\n')\nprint(sample_submission['PredictionString'][3])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:19.274352Z","iopub.execute_input":"2021-06-20T07:15:19.274668Z","iopub.status.idle":"2021-06-20T07:15:19.284963Z","shell.execute_reply.started":"2021-06-20T07:15:19.274636Z","shell.execute_reply":"2021-06-20T07:15:19.284119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsample_submission.to_csv('submission.csv', index = False)\n\n#check\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T07:15:19.286728Z","iopub.execute_input":"2021-06-20T07:15:19.287128Z","iopub.status.idle":"2021-06-20T07:15:19.303055Z","shell.execute_reply.started":"2021-06-20T07:15:19.287085Z","shell.execute_reply":"2021-06-20T07:15:19.301949Z"},"trusted":true},"execution_count":null,"outputs":[]}]}