{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore')\nimport time\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\npd.set_option('display.float_format', lambda x: '%.3f' % x)\nimport numpy as np\nimport json\nimport os\nimport random\nimport string\nimport re\nfrom functools import partial\nimport nltk\n\n\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"id":"Qx3vyCNsUUyv","outputId":"678cadb9-b582-4394-8f98-21e1aafdc407","execution":{"iopub.status.busy":"2021-06-16T13:11:49.489042Z","iopub.execute_input":"2021-06-16T13:11:49.489434Z","iopub.status.idle":"2021-06-16T13:11:50.823163Z","shell.execute_reply.started":"2021-06-16T13:11:49.489334Z","shell.execute_reply":"2021-06-16T13:11:50.822341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')#.sample(frac = 0.3)\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"id":"5qJzNYv_9TMN","execution":{"iopub.status.busy":"2021-06-16T13:11:50.824496Z","iopub.execute_input":"2021-06-16T13:11:50.824828Z","iopub.status.idle":"2021-06-16T13:11:50.939845Z","shell.execute_reply.started":"2021-06-16T13:11:50.824794Z","shell.execute_reply":"2021-06-16T13:11:50.939003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"id":"6hBTRxpyLHc-","outputId":"f15f8aee-e480-444e-f6ff-756c64c1cc90","execution":{"iopub.status.busy":"2021-06-16T13:11:50.941329Z","iopub.execute_input":"2021-06-16T13:11:50.941656Z","iopub.status.idle":"2021-06-16T13:11:50.964887Z","shell.execute_reply.started":"2021-06-16T13:11:50.941619Z","shell.execute_reply":"2021-06-16T13:11:50.963815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path = train_files_path, output = 'text'):\n    json_path = os.path.join(train_files_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:11:51.977091Z","iopub.execute_input":"2021-06-16T13:11:51.977396Z","iopub.status.idle":"2021-06-16T13:11:51.984077Z","shell.execute_reply.started":"2021-06-16T13:11:51.977367Z","shell.execute_reply":"2021-06-16T13:11:51.983031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = train_df['Id'].apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:11:52.619676Z","iopub.execute_input":"2021-06-16T13:11:52.620035Z","iopub.status.idle":"2021-06-16T13:12:48.254907Z","shell.execute_reply.started":"2021-06-16T13:11:52.620006Z","shell.execute_reply":"2021-06-16T13:12:48.254047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"id":"h2yjGR3SqZj0","outputId":"12c957e3-f3d0-466d-baa9-b4de0ef9db0a","execution":{"iopub.status.busy":"2021-06-16T13:12:48.256271Z","iopub.execute_input":"2021-06-16T13:12:48.256612Z","iopub.status.idle":"2021-06-16T13:12:48.262999Z","shell.execute_reply.started":"2021-06-16T13:12:48.256579Z","shell.execute_reply":"2021-06-16T13:12:48.262017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label'].nunique()","metadata":{"id":"T71m8S58qZj0","outputId":"55db566d-3c3b-46fa-9ddb-7798789357a9","execution":{"iopub.status.busy":"2021-06-16T13:12:48.265134Z","iopub.execute_input":"2021-06-16T13:12:48.265564Z","iopub.status.idle":"2021-06-16T13:12:48.275850Z","shell.execute_reply.started":"2021-06-16T13:12:48.265475Z","shell.execute_reply":"2021-06-16T13:12:48.274729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text_len'] = train_df['text'].str.len()\n","metadata":{"id":"HHBdv4GNYB1h","execution":{"iopub.status.busy":"2021-06-16T13:12:48.277717Z","iopub.execute_input":"2021-06-16T13:12:48.278147Z","iopub.status.idle":"2021-06-16T13:12:48.299150Z","shell.execute_reply.started":"2021-06-16T13:12:48.278107Z","shell.execute_reply":"2021-06-16T13:12:48.298407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len_text = int(train_df['text_len'].quantile(0.90))","metadata":{"id":"Bot59SzrYG99","execution":{"iopub.status.busy":"2021-06-16T13:12:48.300376Z","iopub.execute_input":"2021-06-16T13:12:48.300757Z","iopub.status.idle":"2021-06-16T13:12:48.307688Z","shell.execute_reply.started":"2021-06-16T13:12:48.300723Z","shell.execute_reply":"2021-06-16T13:12:48.306855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len_text","metadata":{"id":"mMonztP-gFu3","outputId":"54a82d87-fbe1-4af1-fba7-6f6e2619da50","execution":{"iopub.status.busy":"2021-06-16T13:12:48.309153Z","iopub.execute_input":"2021-06-16T13:12:48.309628Z","iopub.status.idle":"2021-06-16T13:12:48.317430Z","shell.execute_reply.started":"2021-06-16T13:12:48.309594Z","shell.execute_reply":"2021-06-16T13:12:48.316299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_type(word):\n    letters = len([x for x in word if x.isalpha()])\n    numbers = len([x for x in word if x.isnumeric()])\n    return letters*numbers\n\ndef clean_text(text):\n    '''Clean text by removing unnecessary characters and altering the format of words.'''\n\n    text = text.lower()\n    \n    text = re.sub(r\"i'm\", \"i am\", text)\n    text = re.sub(r\"he's\", \"he is\", text)\n    text = re.sub(r\"she's\", \"she is\", text)\n    text = re.sub(r\"it's\", \"it is\", text)\n    text = re.sub(r\"that's\", \"that is\", text)\n    text = re.sub(r\"what's\", \"that is\", text)\n    text = re.sub(r\"where's\", \"where is\", text)\n    text = re.sub(r\"how's\", \"how is\", text)\n    text = re.sub(r\"\\'ll\", \" will\", text)\n    text = re.sub(r\"\\'ve\", \" have\", text)\n    text = re.sub(r\"\\'re\", \" are\", text)\n    text = re.sub(r\"\\'d\", \" would\", text)\n    text = re.sub(r\"\\'re\", \" are\", text)\n    text = re.sub(r\"won't\", \"will not\", text)\n    text = re.sub(r\"can't\", \"cannot\", text)\n    text = re.sub(r\"n't\", \" not\", text)\n    text = re.sub(r\"n'\", \"ng\", text)\n    text = re.sub(r\"'bout\", \"about\", text)\n    text = re.sub(r\"'til\", \"until\", text)\n    text = re.sub(r\"[\\\"#/@;:<>{}`+=~|?,]\", \"\", text)\n    text = re.sub(r\"\\r\", \" \", text)\n    text = re.sub(r\"[.-]\", \" \", text)\n    text = re.sub(r\"[']\", \" \", text)\n    text = re.sub('[^A-Za-z0-9 ]+', '', text)\n    text = \" \".join([x for x in text.split(' ') if (x != '')])\n    text = \" \".join([x for x in text.split(' ') if one_type(x) == 0])\n    \n    return text\n\ndef tagger(decoder_input_sentence):\n    bos = \"<BOS> \"\n    eos = \" <EOS>\"\n    final_target = bos + decoder_input_sentence + eos\n    return final_target","metadata":{"id":"Ci3jfp4Slzg6","execution":{"iopub.status.busy":"2021-06-16T13:12:48.319011Z","iopub.execute_input":"2021-06-16T13:12:48.319456Z","iopub.status.idle":"2021-06-16T13:12:48.334888Z","shell.execute_reply.started":"2021-06-16T13:12:48.319421Z","shell.execute_reply":"2021-06-16T13:12:48.334042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datasets_titles = [x.lower() for x in set(train_df['dataset_title'].unique()).union(set(train_df['dataset_label'].unique()))]","metadata":{"id":"pynWEPsLWYbl","execution":{"iopub.status.busy":"2021-06-16T13:12:48.337845Z","iopub.execute_input":"2021-06-16T13:12:48.338204Z","iopub.status.idle":"2021-06-16T13:12:48.351124Z","shell.execute_reply.started":"2021-06-16T13:12:48.338166Z","shell.execute_reply":"2021-06-16T13:12:48.350257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor index in train_df['Id']:\n    publication_text = train_df[train_df['Id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append(' | '.join(label))\n\ntrain_df['cleaned_label_test'] = labels","metadata":{"id":"oMRi9fKbX1YT","execution":{"iopub.status.busy":"2021-06-16T13:12:48.352524Z","iopub.execute_input":"2021-06-16T13:12:48.352893Z","iopub.status.idle":"2021-06-16T13:17:32.454393Z","shell.execute_reply.started":"2021-06-16T13:12:48.352845Z","shell.execute_reply":"2021-06-16T13:17:32.452833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label_test'].head()","metadata":{"id":"NVGJfyQ7YFXl","outputId":"ed4e5830-8bb3-422c-946d-b14fb9888bbe","execution":{"iopub.status.busy":"2021-06-15T19:04:57.610714Z","iopub.execute_input":"2021-06-15T19:04:57.611077Z","iopub.status.idle":"2021-06-15T19:04:57.617798Z","shell.execute_reply.started":"2021-06-15T19:04:57.611042Z","shell.execute_reply":"2021-06-15T19:04:57.617036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label_test'].apply(lambda x: len(x.split())).max()","metadata":{"id":"1Xngb9b6ZYLa","outputId":"7efa1335-b5b2-42f7-8f7a-eff2761e25e6","execution":{"iopub.status.busy":"2021-06-15T19:04:57.619065Z","iopub.execute_input":"2021-06-15T19:04:57.619673Z","iopub.status.idle":"2021-06-15T19:04:57.649868Z","shell.execute_reply.started":"2021-06-15T19:04:57.619609Z","shell.execute_reply":"2021-06-15T19:04:57.649023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label_test'] = train_df['cleaned_label_test'].apply(lambda x: tagger(x))\n#train_df['cleaned_label'] = train_df['cleaned_label'].apply(lambda x: tagger(x))\ntrain_df['cleaned_text'] = train_df['text'].apply(clean_text)\ntrain_df['cleaned_text'] = train_df['cleaned_text'].str[:max_len_text]","metadata":{"id":"HqhGf_v6lBbW","execution":{"iopub.status.busy":"2021-06-15T19:04:57.651102Z","iopub.execute_input":"2021-06-15T19:04:57.651518Z","iopub.status.idle":"2021-06-15T19:11:42.256318Z","shell.execute_reply.started":"2021-06-15T19:04:57.651483Z","shell.execute_reply":"2021-06-15T19:11:42.255442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'].apply(lambda x: len(x.split())).describe()","metadata":{"id":"4Z3PXnBZkXpr","outputId":"03216734-60ae-4e7e-9235-6b93ea71762a","execution":{"iopub.status.busy":"2021-06-15T19:11:42.257889Z","iopub.execute_input":"2021-06-15T19:11:42.258251Z","iopub.status.idle":"2021-06-15T19:11:52.604287Z","shell.execute_reply.started":"2021-06-15T19:11:42.258213Z","shell.execute_reply":"2021-06-15T19:11:52.603318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label_test'].apply(lambda x: len(x.split(' '))).hist()","metadata":{"id":"N_SJse77LbCn","outputId":"587aea83-c228-4e51-c940-49c3c11cb833","execution":{"iopub.status.busy":"2021-06-15T19:11:52.605732Z","iopub.execute_input":"2021-06-15T19:11:52.606096Z","iopub.status.idle":"2021-06-15T19:11:52.802196Z","shell.execute_reply.started":"2021-06-15T19:11:52.606058Z","shell.execute_reply":"2021-06-15T19:11:52.801236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np  \nimport pandas as pd \nimport re           \nfrom keras.preprocessing.text import Tokenizer \nfrom keras.preprocessing.sequence import pad_sequences\nfrom nltk.corpus import stopwords   \nfrom tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport warnings\npd.set_option(\"display.max_colwidth\", 200)\nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"GMWhnKv9Y5ZV","execution":{"iopub.status.busy":"2021-06-15T19:11:52.803673Z","iopub.execute_input":"2021-06-15T19:11:52.804039Z","iopub.status.idle":"2021-06-15T19:11:57.772563Z","shell.execute_reply.started":"2021-06-15T19:11:52.804001Z","shell.execute_reply":"2021-06-15T19:11:57.771678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"id":"zQoh5TOtLFpA","execution":{"iopub.status.busy":"2021-06-15T19:11:57.774016Z","iopub.execute_input":"2021-06-15T19:11:57.774389Z","iopub.status.idle":"2021-06-15T19:11:57.779531Z","shell.execute_reply.started":"2021-06-15T19:11:57.774351Z","shell.execute_reply":"2021-06-15T19:11:57.777574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len_text=512 \nmax_len_summary=25","metadata":{"id":"t_xVQt36PFeo","execution":{"iopub.status.busy":"2021-06-15T19:11:57.780977Z","iopub.execute_input":"2021-06-15T19:11:57.781472Z","iopub.status.idle":"2021-06-15T19:11:57.78925Z","shell.execute_reply.started":"2021-06-15T19:11:57.781424Z","shell.execute_reply":"2021-06-15T19:11:57.788523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"from sklearn.model_selection import train_test_split\nx_tr,x_val,y_tr,y_val=train_test_split(train_df['cleaned_text'],train_df['cleaned_label_test'],test_size=0.1,random_state=0,shuffle=True) ","metadata":{"id":"ApHDUCeUUU0a","execution":{"iopub.status.busy":"2021-06-14T00:54:17.545435Z","iopub.execute_input":"2021-06-14T00:54:17.545867Z","iopub.status.idle":"2021-06-14T00:54:17.559289Z","shell.execute_reply.started":"2021-06-14T00:54:17.545818Z","shell.execute_reply":"2021-06-14T00:54:17.55788Z"}}},{"cell_type":"code","source":"num_words = 5000","metadata":{"id":"szaCHIAacNO4","execution":{"iopub.status.busy":"2021-06-15T19:11:57.790544Z","iopub.execute_input":"2021-06-15T19:11:57.790904Z","iopub.status.idle":"2021-06-15T19:11:57.79979Z","shell.execute_reply.started":"2021-06-15T19:11:57.790868Z","shell.execute_reply":"2021-06-15T19:11:57.798925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#prepare a tokenizer for reviews on training data\nx_tokenizer = Tokenizer(num_words = num_words)\nx_tokenizer.fit_on_texts(list(train_df['cleaned_text']))\n\n#convert text sequences into integer sequences\nx    =   x_tokenizer.texts_to_sequences(train_df['cleaned_text']) \n\n#padding zero upto maximum length\nx    =   pad_sequences(x,  maxlen=max_len_text, padding='post') \n\nx_voc_size   =  len(x_tokenizer.word_index) +1","metadata":{"id":"B2G93HBNUXaP","execution":{"iopub.status.busy":"2021-06-15T19:11:57.800873Z","iopub.execute_input":"2021-06-15T19:11:57.801123Z","iopub.status.idle":"2021-06-15T19:13:54.160791Z","shell.execute_reply.started":"2021-06-15T19:11:57.801099Z","shell.execute_reply":"2021-06-15T19:13:54.159934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preparing a tokenizer for summary on training data \ny_tokenizer = Tokenizer(num_words = num_words, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{}~\\t\\n')\ny_tokenizer.fit_on_texts(list(train_df['cleaned_label_test']))\n\n#convert summary sequences into integer sequences\ny    =   y_tokenizer.texts_to_sequences(train_df['cleaned_label_test']) \n\n#padding zero upto maximum length\ny    =   pad_sequences(y, maxlen=max_len_summary, padding='post')\n\ny_voc_size  =   len(y_tokenizer.word_index) +1","metadata":{"id":"xyTeAuuUUbDt","execution":{"iopub.status.busy":"2021-06-15T19:13:54.162107Z","iopub.execute_input":"2021-06-15T19:13:54.162463Z","iopub.status.idle":"2021-06-15T19:13:54.758898Z","shell.execute_reply.started":"2021-06-15T19:13:54.162427Z","shell.execute_reply":"2021-06-15T19:13:54.758068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras","metadata":{"id":"cEyPDF42aJG6","execution":{"iopub.status.busy":"2021-06-15T19:13:54.760167Z","iopub.execute_input":"2021-06-15T19:13:54.760656Z","iopub.status.idle":"2021-06-15T19:13:54.764298Z","shell.execute_reply.started":"2021-06-15T19:13:54.760617Z","shell.execute_reply":"2021-06-15T19:13:54.763401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.keras.layers import Layer\nfrom tensorflow.python.keras import backend as K\n\n\nclass AttentionLayer(Layer):\n    \"\"\"\n    This class implements Bahdanau attention (https://arxiv.org/pdf/1409.0473.pdf).\n    There are three sets of weights introduced W_a, U_a, and V_a\n     \"\"\"\n\n    def __init__(self, **kwargs):\n        super(AttentionLayer, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert isinstance(input_shape, list)\n        # Create a trainable weight variable for this layer.\n\n        self.W_a = self.add_weight(name='W_a',\n                                   shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),\n                                   initializer='uniform',\n                                   trainable=True)\n        self.U_a = self.add_weight(name='U_a',\n                                   shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),\n                                   initializer='uniform',\n                                   trainable=True)\n        self.V_a = self.add_weight(name='V_a',\n                                   shape=tf.TensorShape((input_shape[0][2], 1)),\n                                   initializer='uniform',\n                                   trainable=True)\n\n        super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end\n\n    def call(self, inputs, verbose=False):\n        \"\"\"\n        inputs: [encoder_output_sequence, decoder_output_sequence]\n        \"\"\"\n        assert type(inputs) == list\n        encoder_out_seq, decoder_out_seq = inputs\n        if verbose:\n            print('encoder_out_seq>', encoder_out_seq.shape)\n            print('decoder_out_seq>', decoder_out_seq.shape)\n\n        def energy_step(inputs, states):\n            \"\"\" Step function for computing energy for a single decoder state\n            inputs: (batchsize * 1 * de_in_dim)\n            states: (batchsize * 1 * de_latent_dim)\n            \"\"\"\n\n            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n\n            \"\"\" Some parameters required for shaping tensors\"\"\"\n            en_seq_len, en_hidden = encoder_out_seq.shape[1], encoder_out_seq.shape[2]\n            de_hidden = inputs.shape[-1]\n\n            \"\"\" Computing S.Wa where S=[s0, s1, ..., si]\"\"\"\n            # <= batch size * en_seq_len * latent_dim\n            W_a_dot_s = K.dot(encoder_out_seq, self.W_a)\n\n            \"\"\" Computing hj.Ua \"\"\"\n            U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim\n            if verbose:\n                print('Ua.h>', U_a_dot_h.shape)\n\n            \"\"\" tanh(S.Wa + hj.Ua) \"\"\"\n            # <= batch_size*en_seq_len, latent_dim\n            Ws_plus_Uh = K.tanh(W_a_dot_s + U_a_dot_h)\n            if verbose:\n                print('Ws+Uh>', Ws_plus_Uh.shape)\n\n            \"\"\" softmax(va.tanh(S.Wa + hj.Ua)) \"\"\"\n            # <= batch_size, en_seq_len\n            e_i = K.squeeze(K.dot(Ws_plus_Uh, self.V_a), axis=-1)\n            # <= batch_size, en_seq_len\n            e_i = K.softmax(e_i)\n\n            if verbose:\n                print('ei>', e_i.shape)\n\n            return e_i, [e_i]\n\n        def context_step(inputs, states):\n            \"\"\" Step function for computing ci using ei \"\"\"\n\n            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n\n            # <= batch_size, hidden_size\n            c_i = K.sum(encoder_out_seq * K.expand_dims(inputs, -1), axis=1)\n            if verbose:\n                print('ci>', c_i.shape)\n            return c_i, [c_i]\n\n        fake_state_c = K.sum(encoder_out_seq, axis=1)\n        fake_state_e = K.sum(encoder_out_seq, axis=2)  # <= (batch_size, enc_seq_len, latent_dim\n\n        \"\"\" Computing energy outputs \"\"\"\n        # e_outputs => (batch_size, de_seq_len, en_seq_len)\n        last_out, e_outputs, _ = K.rnn(\n            energy_step, decoder_out_seq, [fake_state_e],\n        )\n\n        \"\"\" Computing context vectors \"\"\"\n        last_out, c_outputs, _ = K.rnn(\n            context_step, e_outputs, [fake_state_c],\n        )\n\n        return c_outputs, e_outputs\n\n    def compute_output_shape(self, input_shape):\n        \"\"\" Outputs produced by the layer \"\"\"\n        return [\n            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),\n            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))\n        ]","metadata":{"id":"oSG52A8Pb8Mx","execution":{"iopub.status.busy":"2021-06-15T19:13:54.76556Z","iopub.execute_input":"2021-06-15T19:13:54.766061Z","iopub.status.idle":"2021-06-15T19:13:54.786857Z","shell.execute_reply.started":"2021-06-15T19:13:54.766023Z","shell.execute_reply":"2021-06-15T19:13:54.786055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K \nK.clear_session() \nlatent_dim = 256\ndrop_rate = 0.5\n\n# Encoder \nencoder_inputs = Input(shape=(max_len_text,)) \nenc_emb = Embedding(x_voc_size, latent_dim, trainable = True)(encoder_inputs) \n\n#LSTM 1 \nencoder_lstm1 = LSTM(latent_dim,return_sequences = True, return_state = True)\nencoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\nencoder_output1 = Dropout(drop_rate)(encoder_output1)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output1 = leakyrelu(encoder_output1)\n\n#LSTM 2 \nencoder_lstm2 = LSTM(latent_dim,return_sequences = True,return_state = True) \nencoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1) \nencoder_output2 = Dropout(drop_rate)(encoder_output2)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output2 = leakyrelu(encoder_output2)\n\n#LSTM 3 \nencoder_lstm3 = LSTM(latent_dim, return_state = True, return_sequences = True) \nencoder_output3, state_h3, state_c3 = encoder_lstm3(encoder_output2) \nencoder_output3 = Dropout(drop_rate)(encoder_output3)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output3 = leakyrelu(encoder_output3)\n\n#LSTM 4\nencoder_lstm4 = LSTM(latent_dim, return_state = True, return_sequences = True) \nencoder_output4, state_h4, state_c4 = encoder_lstm4(encoder_output3) \nencoder_output4 = Dropout(drop_rate)(encoder_output4)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output4 = leakyrelu(encoder_output4)\n\n#LSTM 5\nencoder_lstm5 = LSTM(latent_dim, return_state = True, return_sequences = True) \nencoder_output5, state_h5, state_c5 = encoder_lstm5(encoder_output4) \nencoder_output5 = Dropout(drop_rate)(encoder_output5)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output5 = leakyrelu(encoder_output5)\n\n#LSTM 6\nencoder_lstm6 = LSTM(latent_dim, return_state = True, return_sequences = True) \nencoder_output6, state_h6, state_c7 = encoder_lstm6(encoder_output5) \nencoder_output6 = Dropout(drop_rate)(encoder_output6)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\nencoder_output6 = leakyrelu(encoder_output6)\n\n#LSTM 7\nencoder_lstm7 = LSTM(latent_dim, return_state = True, return_sequences = True) \nencoder_outputs, state_h, state_c = encoder_lstm7(encoder_output6) \nencoder_outputs = Dropout(drop_rate)(encoder_outputs)\n\n# Set up the decoder. \ndecoder_inputs = Input(shape=(None,)) \ndec_emb_layer = Embedding(y_voc_size, latent_dim,trainable=True) \ndec_emb = dec_emb_layer(decoder_inputs) \n\n#LSTM 1 using encoder_states as initial state\ndecoder_lstm1 = LSTM(latent_dim, return_sequences=True, return_state=True) \ndecoder_output1,decoder_fwd_state1, decoder_back_state1 = decoder_lstm1(dec_emb,initial_state=[state_h, state_c]) \ndecoder_output1 = Dropout(drop_rate)(decoder_output1)\n\nleakyrelu = tf.keras.layers.LeakyReLU()\ndecoder_outputs = leakyrelu(decoder_output1)\n\n#LSTM 2\n#decoder_lstm2 = LSTM(latent_dim, return_state = True, return_sequences = True) \n#decoder_output2, state_h2, state_c2 = decoder_lstm2(decoder_output1) \n#decoder_output2 = Dropout(drop_rate)(decoder_output2)\n\n#leakyrelu = tf.keras.layers.LeakyReLU()\n#decoder_outputs = leakyrelu(decoder_output2)\n\n#LSTM 3\n#decoder_lstm3 = LSTM(latent_dim, return_state = True, return_sequences = True) \n#decoder_output3, state_h3, state_c3 = decoder_lstm3(decoder_output2) \n#decoder_output3 = Dropout(drop_rate)(decoder_output3)\n\n#leakyrelu = tf.keras.layers.LeakyReLU()\n#decoder_output3 = leakyrelu(decoder_output3)\n\n#LSTM 4\n#decoder_lstm4 = LSTM(latent_dim, return_state = True, return_sequences = True) \n#decoder_outputs, state_h4, state_c4 = decoder_lstm4(decoder_output3) \n#decoder_outputs = Dropout(drop_rate)(decoder_outputs)\n\n\n#Attention Layer\nattn_layer = AttentionLayer(name='attention_layer') \nattn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])\n#attn_out = Dropout(drop_rate)(attn_out)\n\n\n# Concat attention output and decoder LSTM output \ndecoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attn_out])\n\n#Dense layer\ndecoder_dense = TimeDistributed(Dense(y_voc_size, activation='softmax')) \ndecoder_outputs = decoder_dense(decoder_concat_input) \n\n# Define the model\nmodel = Model([encoder_inputs, decoder_inputs], decoder_outputs) \nmodel.summary()","metadata":{"id":"sc0F7IetUdku","outputId":"cd85f53f-0650-45a0-f611-a9cf5f4e051b","execution":{"iopub.status.busy":"2021-06-15T20:30:55.680355Z","iopub.execute_input":"2021-06-15T20:30:55.680681Z","iopub.status.idle":"2021-06-15T20:30:58.692581Z","shell.execute_reply.started":"2021-06-15T20:30:55.680653Z","shell.execute_reply":"2021-06-15T20:30:58.691584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')","metadata":{"id":"On4CQXHIYsK3","execution":{"iopub.status.busy":"2021-06-15T20:30:58.696588Z","iopub.execute_input":"2021-06-15T20:30:58.698767Z","iopub.status.idle":"2021-06-15T20:30:58.715456Z","shell.execute_reply.started":"2021-06-15T20:30:58.698726Z","shell.execute_reply":"2021-06-15T20:30:58.714594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit([x,y[:,:-1]], y.reshape(y.shape[0],y.shape[1], 1)[:,1:] ,epochs=10,\n                  batch_size=32, validation_split = 0.2)","metadata":{"id":"Ms1FgFCNYzb6","outputId":"ca34ab3a-c927-4bf3-f17c-29e93240076a","execution":{"iopub.status.busy":"2021-06-15T20:30:58.71983Z","iopub.execute_input":"2021-06-15T20:30:58.721899Z","iopub.status.idle":"2021-06-15T21:22:23.21986Z","shell.execute_reply.started":"2021-06-15T20:30:58.72186Z","shell.execute_reply":"2021-06-15T21:22:23.216855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot\npyplot.plot(history.history['loss'], label='train') \npyplot.plot(history.history['val_loss'], label='test') \npyplot.legend()\npyplot.show()","metadata":{"id":"xi-iL-KAZPJw","outputId":"ea5736eb-ae68-4458-f208-de304c2286b0","execution":{"iopub.status.busy":"2021-06-15T20:22:05.037068Z","iopub.execute_input":"2021-06-15T20:22:05.03745Z","iopub.status.idle":"2021-06-15T20:22:05.16507Z","shell.execute_reply.started":"2021-06-15T20:22:05.037407Z","shell.execute_reply":"2021-06-15T20:22:05.164289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reverse_target_word_index=y_tokenizer.index_word \nreverse_source_word_index=x_tokenizer.index_word \ntarget_word_index=y_tokenizer.word_index","metadata":{"id":"zQ1Ih4u-c-tV","execution":{"iopub.status.busy":"2021-06-15T20:24:54.555517Z","iopub.execute_input":"2021-06-15T20:24:54.555852Z","iopub.status.idle":"2021-06-15T20:24:54.561547Z","shell.execute_reply.started":"2021-06-15T20:24:54.555821Z","shell.execute_reply":"2021-06-15T20:24:54.560584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# encoder inference\nencoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h5, state_c5])\n\n# decoder inference\n# Below tensors will hold the states of the previous time step\ndecoder_state_input_h = Input(shape=(latent_dim,))\ndecoder_state_input_c = Input(shape=(latent_dim,))\ndecoder_hidden_state_input = Input(shape=(max_len_text,latent_dim))\n\n# Get the embeddings of the decoder sequence\ndec_emb2= dec_emb_layer(decoder_inputs)\n\n# To predict the next word in the sequence, set the initial states to the states from the previous time step\ndecoder_outputs2, state_h2, state_c2 = decoder_lstm1(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n\n#attention inference\nattn_out_inf, attn_states_inf = attn_layer([decoder_hidden_state_input, decoder_outputs2])\ndecoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_outputs2, attn_out_inf])\n\n# A dense softmax layer to generate prob dist. over the target vocabulary\ndecoder_outputs2 = decoder_dense(decoder_inf_concat)\n\n# Final decoder model\ndecoder_model = Model(\n[decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n[decoder_outputs2] + [state_h2, state_c2])","metadata":{"id":"fF3XFE3wdbs5","execution":{"iopub.status.busy":"2021-06-15T20:24:54.790006Z","iopub.execute_input":"2021-06-15T20:24:54.790375Z","iopub.status.idle":"2021-06-15T20:24:55.108098Z","shell.execute_reply.started":"2021-06-15T20:24:54.790343Z","shell.execute_reply":"2021-06-15T20:24:55.107228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_sequence(input_seq):\n    # Encode the input as state vectors.\n    e_out, e_h, e_c = encoder_model.predict(input_seq)\n\n    # Generate empty target sequence of length 1.\n    target_seq = np.zeros((1,1))\n\n    # Chose the 'start' word as the first word of the target sequence\n    target_seq[0, 0] = target_word_index['bos']\n\n    stop_condition = False\n    decoded_sentence = ''\n    while not stop_condition:\n        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n\n        # Sample a token\n        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n        sampled_token = reverse_target_word_index[sampled_token_index]\n        #print(sampled_token)\n        #print(stop_condition)\n        #print(sampled_token == 'eos')\n        if (sampled_token!='eos'):\n            decoded_sentence += ' '+sampled_token\n\n            # Exit condition: either hit max length or find stop word.\n        if (sampled_token == 'eos') or (len(decoded_sentence.split()) >= (max_len_summary-1)):\n            stop_condition = True\n\n        # Update the target sequence (of length 1).\n        target_seq = np.zeros((1,1))\n        target_seq[0, 0] = sampled_token_index\n\n        # Update internal states\n        e_h, e_c = h, c\n\n    return decoded_sentence","metadata":{"id":"Lkn7Fi7KdfEZ","execution":{"iopub.status.busy":"2021-06-15T20:24:55.109572Z","iopub.execute_input":"2021-06-15T20:24:55.109937Z","iopub.status.idle":"2021-06-15T20:24:55.117882Z","shell.execute_reply.started":"2021-06-15T20:24:55.10988Z","shell.execute_reply":"2021-06-15T20:24:55.116829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"id":"6_SpadwFiJCT","execution":{"iopub.status.busy":"2021-06-15T20:24:55.301081Z","iopub.execute_input":"2021-06-15T20:24:55.301442Z","iopub.status.idle":"2021-06-15T20:24:55.305969Z","shell.execute_reply.started":"2021-06-15T20:24:55.301411Z","shell.execute_reply":"2021-06-15T20:24:55.305164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:55.9874Z","iopub.execute_input":"2021-06-15T20:24:55.987753Z","iopub.status.idle":"2021-06-15T20:24:55.999064Z","shell.execute_reply.started":"2021-06-15T20:24:55.987721Z","shell.execute_reply":"2021-06-15T20:24:55.998221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:56.328035Z","iopub.execute_input":"2021-06-15T20:24:56.328408Z","iopub.status.idle":"2021-06-15T20:24:56.340835Z","shell.execute_reply.started":"2021-06-15T20:24:56.328362Z","shell.execute_reply":"2021-06-15T20:24:56.339941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].apply(lambda x: read_append_return(x, test_files_path))","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:56.968191Z","iopub.execute_input":"2021-06-15T20:24:56.96852Z","iopub.status.idle":"2021-06-15T20:24:56.978755Z","shell.execute_reply.started":"2021-06-15T20:24:56.968491Z","shell.execute_reply":"2021-06-15T20:24:56.977903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['cleaned_text'] = sample_sub['text'].apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:57.402948Z","iopub.execute_input":"2021-06-15T20:24:57.403297Z","iopub.status.idle":"2021-06-15T20:24:57.531591Z","shell.execute_reply.started":"2021-06-15T20:24:57.403263Z","shell.execute_reply":"2021-06-15T20:24:57.53069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_seqs = x_tokenizer.texts_to_sequences(sample_sub['cleaned_text'])\nsub_seqs = pad_sequences(sub_seqs, maxlen=max_len_text, padding='post') ","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:58.272175Z","iopub.execute_input":"2021-06-15T20:24:58.272527Z","iopub.status.idle":"2021-06-15T20:24:58.304616Z","shell.execute_reply.started":"2021-06-15T20:24:58.272497Z","shell.execute_reply":"2021-06-15T20:24:58.30379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted = []\nfor i in range(len(sub_seqs)):\n    decoded_sequence = decode_sequence(sub_seqs[i].reshape(1, max_len_text)).strip().replace(' | ','|')\n    print(decoded_sequence)\n    predicted.append(decoded_sequence)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:24:58.92363Z","iopub.execute_input":"2021-06-15T20:24:58.923957Z","iopub.status.idle":"2021-06-15T20:25:01.766674Z","shell.execute_reply.started":"2021-06-15T20:24:58.923918Z","shell.execute_reply":"2021-06-15T20:25:01.765721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['PredictionString'] = predicted","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:25:04.885768Z","iopub.execute_input":"2021-06-15T20:25:04.886098Z","iopub.status.idle":"2021-06-15T20:25:04.890442Z","shell.execute_reply.started":"2021-06-15T20:25:04.886066Z","shell.execute_reply":"2021-06-15T20:25:04.889344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = sample_sub.drop(['text','cleaned_text'], axis = 1)\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:25:05.129912Z","iopub.execute_input":"2021-06-15T20:25:05.130263Z","iopub.status.idle":"2021-06-15T20:25:05.140619Z","shell.execute_reply.started":"2021-06-15T20:25:05.13023Z","shell.execute_reply":"2021-06-15T20:25:05.139659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv('submission.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}