{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport numpy as np\nimport pandas as pd\nimport nltk\n#import spacy\nfrom tqdm import tqdm\nimport copy\n#from collections import Counter\nimport random\nrandom.seed(42)\n#import requests, zipfile, io\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\n#nlp = spacy.load('en', disable=['parser', 'ner'])\n#nlp.max_length = 4000000\n\n#from sklearn.preprocessing import LabelEncoder\nfrom sklearn.feature_extraction.text import TfidfTransformer\n#from sklearn.preprocessing import StandardScaler\n#from collections import defaultdict\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.linear_model import SGDClassifier, LogisticRegression\nfrom sklearn.pipeline import Pipeline\n#from sklearn.model_selection import GridSearchCV\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Bidirectional, Dense, Input, LSTM, Dropout, Embedding, BatchNormalization, ReLU\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.optimizers import Adam\n#from tensorflow.keras.regularizers import l2\n#from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n\nfrom transformers import TFDistilBertModel, DistilBertConfig\nfrom transformers import DistilBertTokenizer\n\n#from sklearn.model_selection import train_test_split\n\nfrom itertools import groupby\nfrom operator import itemgetter\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Helper Functions for Cleaning**\n","metadata":{}},{"cell_type":"code","source":"#import data\ndef read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n        f.close()\n        del json_decode\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\n\n#split sentences\ndef sentence_creator(string):\n    st = string.replace('\\n', ' ')\n    st = st.replace(' . ', ' ')\n    #st = st.replace('NSF.', 'NSF')\n    #st = st.replace('Foundation.', 'Foundation')\n    #st = st.replace('Cov-2.', 'Cov-2')\n    #regex = re.compile('([A-Z]\\w+[-[\\d[.]+[A-Z][.]|[A-Z]\\w+[-[\\d[.]+|[A-Z]+[.])', re.S)\n    regex = re.compile('([A-Z]+[.]+[a-z]\\w+|[A-Z]\\w+[-[\\d[.]+[A-Z][.]\\s|[A-Z]\\w+[-[\\d[.]+\\s)', re.S)\n    clean = regex.sub(lambda m: m.group().replace('.', ' '), st)\n    clean = regex.sub(lambda m: m.group().replace('-', ' '), clean)\n    sent = re.split('(?<!\\w\\.\\w.)(?<![A-Z][a-z]\\.)(?<=\\.|\\?)\\s', clean)\n    #sent = string.split('.\\n')\n    return sent\n\n#clean sentences\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###combine the labels into lists in the df\n#we are looking to reduce the df to the number of papers present in the train dir\ndataset_df = train_df.groupby('dataset_title').agg({'dataset_label':'unique', 'cleaned_label':'unique'}).reset_index()\n\nfor row in dataset_df.itertuples():\n    labels = [*row.cleaned_label.tolist()]\n    labels = set([lab.lower() for lab in labels])\n    dataset_df.loc[row.Index, 'labels'] = '|'.join(labels)\n\ndataset_df.labels = dataset_df.labels.apply(lambda x: x.split('|'))\ndataset_df.pop('cleaned_label')\ndataset_df.pop('dataset_label')\n\nknown_labels = [item for sublist in dataset_df.labels.to_list() for item in sublist]\nknown_labels = list(set(known_labels))\n\ntrain_df = train_df.merge(dataset_df, on='dataset_title')\ndel dataset_df\n\ntrain_df.drop_duplicates(subset='Id', inplace=True)\ntrain_df.reset_index(drop=True, inplace=True)\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split sentences\ntrain_df['sentences'] = train_df.text.progress_apply(sentence_creator)\ndel train_df['text']\n#clean\ntrain_df['clean_sentences'] = [list(map(clean_text, x)) for x in tqdm(train_df['sentences'])]\ndel train_df['sentences']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_files_path\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df['dataset_title']\ndel train_df['dataset_label']\ndel train_df['cleaned_label']\ndel train_df['pub_title']\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets now label the data and create a new column\n#we will make a tuple with: (sentence, label)\n#label of 1 indicates sentence has the dataset\n#label of 0 indicates it does not \ndef binary_labeler(column):\n        labels = []\n        for i in column[2]:\n                #if any(t in i for t in column[8]) == True:\n                if any(t in i for t in known_labels) == True:\n                    labels.append((i, 1))\n                else:\n                    labels.append((i,0))\n        return labels\n     \ntrain_df['binary_label'] = train_df.progress_apply(binary_labeler, axis=1)        \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del binary_labeler\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#expand the dataset so each row is 1 sentence\ntrain_df = train_df.explode('binary_label').reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#expand the tuple into 2 columns: single_sentence and y_label\ntrain_df[['single_sentence', 'y_label']] = pd.DataFrame(train_df['binary_label'].tolist(), index=train_df.index)\ntrain_df = train_df[train_df.single_sentence != '']\ntrain_df = train_df[train_df['single_sentence'].apply(lambda x: len(x) >= 3)]\n\ndel train_df['binary_label']\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets clean a bit more, and check how balanced the data is \ntrain_df = train_df.drop(train_df[(train_df['y_label'] == 0) & (train_df['single_sentence'].apply(lambda x: len(x) <= 20) )].index)\ntrain_df.groupby('y_label').count() #/ len(train_df)\n#so there are only 1.46% sentences labeled as a 1\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we need to undersample the non dataset sentences to make the dataset more equal\ny = copy.deepcopy(train_df['y_label'])\nx = copy.deepcopy(train_df)\n\ndef sample_together(n, X, y):\n    rows = random.sample(np.arange(0,len(X.index)).tolist(),n)\n    return X.iloc[rows,], y.iloc[rows,]\n\ndef undersample(X, y, under = 1):\n    y_min = y[y == under]\n    y_max = y[y != under]\n    X_min = X.filter(y_min.index,axis = 0)\n    X_max = X.filter(y_max.index,axis = 0)\n\n    X_under, y_under = sample_together(int((len(y_min.index) * 0.8) / 0.2), X_max, y_max)\n    \n    X = pd.concat([X_under, X_min])\n    y = pd.concat([y_under, y_min])\n    return X, y\n\nx, y_train = undersample(x, y)\n\ntrain_balanced_df = x\ntrain_balanced_df.groupby('y_label').count() / len(train_balanced_df)\n\ndel x\ndel y\ndel y_train\ndel train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_balanced_df['clean_sentences']\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_balanced_df.reset_index(inplace=True, drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#need to remove words from sentences including:\nstops = ['b','c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', \n         'l', 'm', 'n', 'o', 'p', 'q', 'r', 't',\n         'u', 'v', 'w', 'x', 'y', 'z']\n\ndef remove_stops(x):\n    y = ' '.join([word for word in x.split() if word not in stops])\n    return y\n\ntrain_balanced_df['single_sentence'] = train_balanced_df.single_sentence.progress_apply(remove_stops)\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets get the top most used words for the datasets\n#and count vectorize the sentences based on the keywords\nkeyword_df = pd.Series(known_labels)\n#keyword_df = keyword_df.drop_duplicates().reset_index(drop=True)\nfor i in range(len(keyword_df)):\n    keyword_df[i] = keyword_df[i].split(' ')\nkeyword_df = keyword_df.explode()#.drop_duplicates().reset_index(drop=True)\nkeyword_df = keyword_df.map(remove_stops)\n\nstop_words = set(nltk.corpus.stopwords.words('english'))\ndef remove_nltk_stops(x):\n    y = ' '.join([word for word in x.split() if word not in stop_words])\n    return y\nkeyword_df = keyword_df.map(remove_nltk_stops)\n\nkeyword_df = keyword_df[keyword_df != '']\n\nkeyword_vc = pd.DataFrame({\"Count\": keyword_df.value_counts()}).reset_index()\nkeyword_vc.columns = ['keyword', 'count']\n\nkeyword_vc = keyword_vc['keyword'].to_list()\n\ndel keyword_df\ndel stop_words\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#prepare main dataset for deep learning\nnum_words = 25000\nmax_padding = 128\n\nsentence_list = train_balanced_df['single_sentence'].to_list()\n\n\ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words = num_words,\n                                                         oov_token = '<OOV>')\ntokenizer.fit_on_texts(sentence_list)\nword_index = tokenizer.word_index\nsequences = tokenizer.texts_to_sequences(sentence_list)\n\nX_sentence = tf.keras.preprocessing.sequence.pad_sequences(sequences, \n                                                         maxlen = max_padding,\n                                                         truncating = 'post',\n                                                         padding = 'post')\n\n\nreverse_index = dict([(value, key) for (key,value) in word_index.items()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sequences\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using this sequence length lets get the original sequences and create our\n#keyword dataset\n#here we will take the sequence and replace non-keys with 0\n#and replace keys with its index in the sentence \ndef sequence_to_text(list_of_indices):\n    # Looking up words in dictionary\n    words = [reverse_index.get(letter) for letter in list_of_indices]\n    return(words)\n\nreverse_sentences = list(map(sequence_to_text, tqdm(X_sentence)))\n            \nfor i in tqdm(range(len(reverse_sentences))):\n    for j in range(len(reverse_sentences[i])):\n        if reverse_sentences[i][j] not in keyword_vc:\n            reverse_sentences[i][j] = 0\n        else:\n            reverse_sentences[i][j] = word_index[reverse_sentences[i][j]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_keyword = reverse_sentences #list(X_keyword.numpy())\n#ids = train_balanced_df['Id']\ny_label = train_balanced_df['y_label']\n\ndataset = pd.DataFrame({\n    #'id': ids,\n    'x_sentence': list(X_sentence),\n    'x_keyword': X_keyword,\n    'sentences': sentence_list,\n    'y_label': y_label})\n\ndataset = dataset.sample(frac=1).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create our final data\n#X_train_sentences = np.stack(dataset['x_sentence'].to_list())\nX_train_keywords = np.stack(dataset['x_keyword'].to_list())\nX_train_tfidf = dataset['sentences'].to_list()\nY_train = dataset['y_label'].to_numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del ids\ndel X_keyword\ndel y_label\ndel dataset\ndel X_sentence\ndel train_balanced_df\ndel sentence_list\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prepare Test Data**","metadata":{}},{"cell_type":"code","source":"def read_append_return_test(filename, train_files_path=test_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n        f.close()\n        del json_decode\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].progress_apply(read_append_return_test)\nsample_sub['sentences'] = sample_sub.text.progress_apply(sentence_creator)\nsample_sub['clean_sentences'] = [list(map(clean_text, x)) for x in tqdm(sample_sub['sentences'])]\n\nsample_sub = sample_sub.explode('clean_sentences').reset_index(drop=True)\n\nsample_sub = sample_sub[sample_sub['clean_sentences'] != '']\nsample_sub = sample_sub[sample_sub['clean_sentences'].apply(lambda x: len(x) >= 3)]\n\nsample_sub['clean_sentences'] = sample_sub['clean_sentences'].progress_apply(remove_stops)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sample_sub['text']\ndel sample_sub['sentences']\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sentence_list = sample_sub['clean_sentences'].to_list()\n\ntest_sequences = tokenizer.texts_to_sequences(test_sentence_list)\n\nX_test_sentences = tf.keras.preprocessing.sequence.pad_sequences(test_sequences, \n                                                         maxlen = max_padding,\n                                                         truncating = 'post',\n                                                         padding = 'post')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reverse_test_sentences = list(map(sequence_to_text, tqdm(X_test_sentences)))\n            \nfor i in tqdm(range(len(reverse_test_sentences))):\n    for j in range(len(reverse_test_sentences[i])):\n        if reverse_test_sentences[i][j] not in keyword_vc:\n            reverse_test_sentences[i][j] = 0\n        else:\n            reverse_test_sentences[i][j] = word_index[reverse_test_sentences[i][j]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_test_sentences\ndel test_sentence_list\ndel word_index\ndel reverse_index\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_keywords = np.stack(reverse_test_sentences) #list(X_keyword.numpy())\n#ids_test = sample_sub['Id']\nX_test_tfidf = sample_sub['clean_sentences'].to_list()\n#X_test_sentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del reverse_sentences\ndel tokenizer\ndel test_sequences\ndel reverse_test_sentences\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MODELS**","metadata":{}},{"cell_type":"code","source":"#MODEL 1\n#bag of words with Support Vector Machine\ntext_clf_svm = Pipeline([('vect', CountVectorizer(ngram_range = (1,2))),\n                      ('tfidf', TfidfTransformer(use_idf=False)),\n                      ('clf-svm', SGDClassifier(loss='log', penalty='l2',\n                                            alpha=1e-5, random_state=42)),\n ])\n\n_ = text_clf_svm.fit(X_train_tfidf, Y_train)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svm_test_preds = text_clf_svm.predict(X_test_tfidf)\nsvm_train_preds = text_clf_svm.predict(X_train_tfidf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del text_clf_svm\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODEL 2\n#bag of words + sequence\n\ndef rnn_classifier(units, dropout, learning_rate, num_layers, input_length=128):\n    inpt = Input(shape=(128,))\n    x = Embedding(num_words+1, 300, input_length=input_length, mask_zero=True)(inpt)\n    x = Bidirectional(LSTM(64, activation = 'relu', ))(x)\n    x = Dropout(dropout)(x)\n    #x = Bidirectional(LSTM(units, activation ='relu'))(x)\n    for _ in range(1, num_layers):\n        x = Dense(units)(x)\n        x = BatchNormalization()(x)\n        x = ReLU()(x)\n        x = Dropout(dropout)(x)\n    x = Dense(1, activation='sigmoid')(x)\n    model = Model(inpt, x)\n    model.compile(loss = 'binary_crossentropy', optimizer = Adam(learning_rate),\n                  metrics=['accuracy'])\n    return model\n\nclass StopEarly(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs={}):\n        if (logs.get('accuracy') > 0.995):\n            print('\\n Stop Training, Accuracy is 99.5%')\n            self.model.stop_training = True\n\ncallbacks = StopEarly()\n\nmodel = rnn_classifier(32, 0.2, 1e-3, 2)\nmodel.fit(X_train_keywords, Y_train, batch_size = 256, epochs = 2, verbose=1,\n          callbacks=[callbacks])\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rnn_test_preds = model.predict(X_test_keywords, verbose=1)\nrnn_train_preds = model.predict(X_train_keywords, verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ndel callbacks\ndel X_test_keywords\ndel X_train_keywords\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODEL 3\n#transformer DistilBERT on entire tokenized sentence\ndistil_bert_token = '../input/distilbertbaseuncased/'\n\ntokenizer = DistilBertTokenizer.from_pretrained(distil_bert_token)\n\n\ndef batch_encode(tokenizer, texts, max_length=max_padding):\n    \"\"\"\"\"\"\"\"\"\n    A function that encodes a batch of texts and returns the texts'\n    corresponding encodings and attention masks that are ready to be fed \n    into a pre-trained transformer model.\n    \n    Input:\n        - tokenizer:   Tokenizer object from the PreTrainedTokenizer Class\n        - texts:       List of strings where each string represents a text\n        - batch_size:  Integer controlling number of texts in a batch\n        - max_length:  Integer controlling max number of words to tokenize in a given text\n    Output:\n        - input_ids:       sequence of texts encoded as a tf.Tensor object\n        - attention_mask:  the texts' attention mask encoded as a tf.Tensor object\n    \"\"\"\"\"\"\"\"\"\n    \n    input_ids = []\n    attention_mask = []\n    \n    for i in tqdm(texts):\n        inputs = tokenizer.encode_plus(i,\n                                             max_length=max_length,\n                                             padding='max_length',\n                                             truncation=True,\n                                             return_attention_mask=True,\n                                             return_token_type_ids=False\n                                             )\n        input_ids.append(inputs['input_ids'])\n        attention_mask.append(inputs['attention_mask'])\n    \n    \n    return np.asarray(input_ids, dtype='int32'), np.asarray(attention_mask, dtype='int32')\n\nX_train_ids, X_train_attention = batch_encode(tokenizer, X_train_tfidf)\n\nX_test_ids, X_test_attention = batch_encode(tokenizer, X_test_tfidf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del tokenizer\ndel distil_bert_token\ndel X_test_tfidf\ndel X_train_tfidf\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distil_bert = distil_bert = '../input/tfdistilbert-base/distilbert-base-uncased-tf_model.h5'\n\nconfig = DistilBertConfig(dropout=0.2, attention_dropout=0.2, n_layers=1)\nconfig.output_hidden_states = False\ntransformer_bert = TFDistilBertModel.from_pretrained(distil_bert, config = config)\n\n\n\ndef transformer_model(transformer, num_layers, units, lr):\n    input_ids_in = tf.keras.layers.Input(shape=(128,), name='input_token', dtype='int32')\n    input_masks_in = tf.keras.layers.Input(shape=(128,), name='masked_token', dtype='int32') \n    \n    embedding_layer = transformer(input_ids_in, attention_mask=input_masks_in)[0]\n    X = Bidirectional(tf.keras.layers.LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(embedding_layer)\n    X = tf.keras.layers.GlobalMaxPool1D()(X)\n    for _ in range(1,num_layers):\n        X = Dense(units, activation='relu')(X)\n        X = Dropout(0.2)(X)\n    X = Dense(1, activation='sigmoid')(X)\n    model = tf.keras.Model(inputs=[input_ids_in, input_masks_in], outputs = X)\n    \n    model.compile(optimizer = Adam(lr), loss = 'binary_crossentropy', \n                  metrics = ['accuracy'])\n    \n    return model\n\n\n\nmodel_distil = transformer_model(transformer_bert, 2, 32, 1e-3)\nfor layer in model_distil.layers[:3]:\n  layer.trainable = False\n\n\ntrain_history1 = model_distil.fit(\n    x = [X_train_ids, X_train_attention],\n    y = Y_train,\n    epochs = 1,\n    batch_size = 64,\n    verbose=1\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_test_preds = model_distil.predict([X_test_ids, X_test_attention], verbose=1)\nbert_train_preds = model_distil.predict([X_train_ids, X_train_attention], verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del distil_bert\ndel model_distil\ndel X_train_ids\ndel X_train_attention\ndel X_test_ids\ndel X_test_attention\n\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Ensemble Models**","metadata":{}},{"cell_type":"code","source":"train_preds = pd.DataFrame({'svm': svm_train_preds, \n                            'preds':rnn_train_preds.reshape((len(rnn_train_preds),)), \n                            'bert':bert_train_preds.reshape((len(rnn_train_preds),))}).values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#logistic regression\nfinal_model = LogisticRegression(random_state=42, multi_class='ovr')\nfinal_model.fit(train_preds, Y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = pd.DataFrame({'svm': svm_test_preds, \n                            'preds':rnn_test_preds.reshape((len(rnn_test_preds),)), \n                            'bert':bert_test_preds.reshape((len(rnn_test_preds),))}).values\n\nfinal_test_preds = final_model.predict(test_preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_preds\ndel test_preds\ndel final_model\ndel svm_test_preds\ndel rnn_test_preds\ndel bert_test_preds\ndel svm_train_preds\ndel rnn_train_preds\ndel bert_train_preds\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for i in range(len(train_preds)):\n#    if train_preds[i] >= 0.5:\n#        train_preds[i] = 1\n#    else:\n#        train_preds[i] = 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prepare Data for Submission**","metadata":{}},{"cell_type":"code","source":"sample_sub['preds'] = final_test_preds\nsample_sub.reset_index(inplace=True, drop=True)\n\n#known_labels\n\ndef dataset_labeler(column):\n    #if column[4] == 1:\n        titles = []\n        sent_split = column.split(' ')\n        #option 1\n        for i in known_labels:\n            if i in column:\n                titles.append(i)\n            \n        #option 2\n        if any(n in column for n in known_labels) == False:\n            words = []\n            for t in keyword_vc:\n                for x in sent_split:\n                    if t == x:\n                        words.append(t)\n            if len(words) == 1:\n                key = []\n                \n                for q, r in enumerate(sent_split):\n                    if r == words[0]:\n                        key.append(q)\n                if len(sent_split) <= 7:\n                    titles.append(' '.join(sent_split))\n                if key[0] == 0 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]:key[0]+7]))\n                if key[0] == 1 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-1:key[0]+6]))\n                if key[0] == 2 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-2:key[0]+5]))\n                if key[0] == 3 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-3:key[0]+4]))\n                if key[0]+1 == len(column) and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-7:key[0]]))\n                if key[0]+1 == len(column)-1 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-6:key[0]+1]))\n                if key[0]+1 == len(column)-2 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-5:key[0]+2]))\n                if key[0]+1 == len(column)-3 and len(sent_split) > 7:\n                    titles.append(' '.join(sent_split[key[0]-4:key[0]+3]))\n                else:\n                    titles.append(' '.join(sent_split[key[0]-3:key[0]+4]))\n            if len(words) == 2:\n                key = []\n                \n                for q, r in enumerate(sent_split):\n                    for g in words:\n                        if r == g:\n                            key.append(q)\n                if len(sent_split) <= 6:\n                    titles.append(' '.join(sent_split))\n                if key[0] == 0 and len(sent_split) > 6:\n                    titles.append(' '.join(sent_split[key[0]:key[1]+5]))\n                if key[0] == 1 and len(sent_split) > 6:\n                    titles.append(' '.join(sent_split[key[0]-1:key[1]+4]))\n                if key[0] == 2 and len(sent_split) > 6:\n                    titles.append(' '.join(sent_split[key[0]-2:key[1]+3]))\n                if key[0] == 3 and len(sent_split) > 6:\n                    titles.append(' '.join(sent_split[key[0]-3:key[1]+2]))\n                if key[0] == 4 and len(sent_split) > 6:\n                    titles.append(' '.join(sent_split[key[0]-4:key[1]+1]))\n                \n            if len(words) > 2:\n                key = []\n                x = []\n                for q, r in enumerate(sent_split):\n                    for g in words:\n                        if r == g:\n                            key.append(q)\n                key = list(set(key))\n                key.sort()\n                #while key[-1] - key[0] > 17:\n                    #del key[-1]\n                for k, g in groupby(enumerate(key), lambda x: x[1]-x[0]):\n                    x.append(list(map(itemgetter(1), g)))\n                    key_sub = max((i) for i in x)\n                titles.append(' '.join(sent_split[key_sub[0]:key_sub[-1]+1]))\n                \n                #dist = [j-i for i, j in zip(key[:-1], key[1:])] \n                \n            #option 3\n                #if any(m in column for m in keyword_vc) == False:\n                    #print('False Positive')\n                    #column[4] = 0\n            \n        for k in range(len(titles)):\n                    if len(titles[k]) < 1:\n                        del titles[k]\n\n        for k in range(len(titles)):\n                    if len(titles[k].split(' ')) <= 2 and any(n in titles[k] for n in known_labels) == False:\n                        del titles[k]\n            \n            \n        titles = list(set(titles))\n        return titles\n\nsample_sub['dataset'] = sample_sub.clean_sentences.progress_apply(dataset_labeler)\n\nfor n in tqdm(range(len(sample_sub))):\n    if sample_sub.preds[n] == 0:\n        sample_sub.dataset[n] = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df = sample_sub.groupby('Id').agg({'Id': 'unique','dataset':'sum'}).reset_index(drop=True)\n\nfor row in preds_df.itertuples():\n    labels = [*row.dataset]\n    labels = set(labels)\n    preds_df.loc[row.Index, 'labels'] = '|'.join(labels)\n    preds_df.loc[row.Index, 'Id'] = ''.join(*row.Id)\n    \npreds_df.pop('dataset')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sample_sub\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df.labels[2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaner_labels = []\n\nmessy_labels = preds_df.labels.to_list()\npreds_df.pop('labels')\n\nfor j in messy_labels:\n    tmp_labels = []\n    if any(n in j for n in known_labels) == True:\n        for i in known_labels:\n            if i in j:\n                tmp_labels.append(i)\n        cleaner_labels.append('|'.join(tmp_labels))\n    if any(n in j for n in known_labels) == False:\n        tmp_labels.append(j)\n        cleaner_labels.append(tmp_labels[0])\n\npreds_df['labels'] = cleaner_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del cleaner_labels\ndel messy_labels\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df.columns = ['Id', 'PredictionString']\nsubmission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsubmission.pop('PredictionString')\nsubmission = submission.merge(preds_df, on='Id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del preds_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv')","metadata":{},"execution_count":null,"outputs":[]}]}