{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Asthetics\nimport warnings\n#warnings.filterwarnings('ignore')\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\npd.set_option('display.float_format', lambda x: '%.3f' % x)\nimport numpy as np\nimport json\nimport os\nimport random\nimport string\nimport re\nfrom functools import partial\nimport nltk\nimport time","metadata":{"id":"Qx3vyCNsUUyv","outputId":"b6cd7ed6-2051-4b55-e5df-4505eed84f17","execution":{"iopub.status.busy":"2021-06-12T16:29:03.86462Z","iopub.execute_input":"2021-06-12T16:29:03.865383Z","iopub.status.idle":"2021-06-12T16:29:05.121876Z","shell.execute_reply.started":"2021-06-12T16:29:03.865283Z","shell.execute_reply":"2021-06-12T16:29:05.121039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"id":"BPY9un0lqZjx","execution":{"iopub.status.busy":"2021-06-12T16:29:05.123203Z","iopub.execute_input":"2021-06-12T16:29:05.123509Z","iopub.status.idle":"2021-06-12T16:29:05.239702Z","shell.execute_reply.started":"2021-06-12T16:29:05.123474Z","shell.execute_reply":"2021-06-12T16:29:05.23877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path = train_files_path, output = 'text'):\n    json_path = os.path.join(train_files_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:29:05.241266Z","iopub.execute_input":"2021-06-12T16:29:05.241576Z","iopub.status.idle":"2021-06-12T16:29:05.250659Z","shell.execute_reply.started":"2021-06-12T16:29:05.241542Z","shell.execute_reply":"2021-06-12T16:29:05.249556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = train_df['Id'].apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:29:05.252439Z","iopub.execute_input":"2021-06-12T16:29:05.252858Z","iopub.status.idle":"2021-06-12T16:30:07.176913Z","shell.execute_reply.started":"2021-06-12T16:29:05.252819Z","shell.execute_reply":"2021-06-12T16:30:07.176062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"id":"h2yjGR3SqZj0","outputId":"4413e16f-d4e8-4c74-887b-632d42b599e4","execution":{"iopub.status.busy":"2021-06-12T16:30:07.178514Z","iopub.execute_input":"2021-06-12T16:30:07.178855Z","iopub.status.idle":"2021-06-12T16:30:07.186982Z","shell.execute_reply.started":"2021-06-12T16:30:07.178821Z","shell.execute_reply":"2021-06-12T16:30:07.185946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_label'].nunique()","metadata":{"id":"T71m8S58qZj0","outputId":"b28cd9a2-a64f-450c-abcd-13256bad34d5","execution":{"iopub.status.busy":"2021-06-12T16:30:07.18875Z","iopub.execute_input":"2021-06-12T16:30:07.189102Z","iopub.status.idle":"2021-06-12T16:30:07.20015Z","shell.execute_reply.started":"2021-06-12T16:30:07.189067Z","shell.execute_reply":"2021-06-12T16:30:07.199238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text_len'] = train_df['text'].str.len()\ntrain_df['label_len'] = train_df['cleaned_label'].str.len()","metadata":{"id":"HHBdv4GNYB1h","execution":{"iopub.status.busy":"2021-06-12T16:30:07.201533Z","iopub.execute_input":"2021-06-12T16:30:07.20197Z","iopub.status.idle":"2021-06-12T16:30:07.234008Z","shell.execute_reply.started":"2021-06-12T16:30:07.201934Z","shell.execute_reply":"2021-06-12T16:30:07.233281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text_len'].quantile(0.90)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:30:07.236254Z","iopub.execute_input":"2021-06-12T16:30:07.236575Z","iopub.status.idle":"2021-06-12T16:30:07.245092Z","shell.execute_reply.started":"2021-06-12T16:30:07.236544Z","shell.execute_reply":"2021-06-12T16:30:07.244005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len_text = int(train_df['text_len'].quantile(0.90))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:30:07.247046Z","iopub.execute_input":"2021-06-12T16:30:07.247419Z","iopub.status.idle":"2021-06-12T16:30:07.254351Z","shell.execute_reply.started":"2021-06-12T16:30:07.247385Z","shell.execute_reply":"2021-06-12T16:30:07.253622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['text_len'] >= train_df['text_len'].quantile(0.90)]","metadata":{"id":"Bot59SzrYG99","outputId":"1d385943-25ca-403b-eb41-488597d50e3c","execution":{"iopub.status.busy":"2021-06-12T16:30:07.255611Z","iopub.execute_input":"2021-06-12T16:30:07.256053Z","iopub.status.idle":"2021-06-12T16:30:07.303027Z","shell.execute_reply.started":"2021-06-12T16:30:07.256021Z","shell.execute_reply":"2021-06-12T16:30:07.302268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_type(word):\n    letters = len([x for x in word if x.isalpha()])\n    numbers = len([x for x in word if x.isnumeric()])\n    return letters*numbers\n\ndef clean_text(text):\n    '''Clean text by removing unnecessary characters and altering the format of words.'''\n\n    text = text.lower()\n    \n    text = re.sub(r\"i'm\", \"i am\", text)\n    text = re.sub(r\"he's\", \"he is\", text)\n    text = re.sub(r\"she's\", \"she is\", text)\n    text = re.sub(r\"it's\", \"it is\", text)\n    text = re.sub(r\"that's\", \"that is\", text)\n    text = re.sub(r\"what's\", \"that is\", text)\n    text = re.sub(r\"where's\", \"where is\", text)\n    text = re.sub(r\"how's\", \"how is\", text)\n    text = re.sub(r\"\\'ll\", \" will\", text)\n    text = re.sub(r\"\\'ve\", \" have\", text)\n    text = re.sub(r\"\\'re\", \" are\", text)\n    text = re.sub(r\"\\'d\", \" would\", text)\n    text = re.sub(r\"\\'re\", \" are\", text)\n    text = re.sub(r\"won't\", \"will not\", text)\n    text = re.sub(r\"can't\", \"cannot\", text)\n    text = re.sub(r\"n't\", \" not\", text)\n    text = re.sub(r\"n'\", \"ng\", text)\n    text = re.sub(r\"'bout\", \"about\", text)\n    text = re.sub(r\"'til\", \"until\", text)\n    text = re.sub(r\"[\\\"#/@;:<>{}`+=~|.!?,]\", \"\", text)\n    text = re.sub(r\"-\", \" \", text)\n    text = re.sub('[^A-Za-z0-9 ]+', '', text)\n    text = \" \".join([x for x in text.split(' ') if (x != '') and (len(x) != 1)])\n    text = \" \".join([x for x in text.split(' ') if one_type(x) == 0])\n    \n    return text\n\ndef tagger(decoder_input_sentence):\n    bos = \"<BOS> \"\n    eos = \" <EOS>\"\n    final_target = bos + decoder_input_sentence + eos\n    return final_target","metadata":{"id":"Ci3jfp4Slzg6","execution":{"iopub.status.busy":"2021-06-12T16:30:07.304275Z","iopub.execute_input":"2021-06-12T16:30:07.304593Z","iopub.status.idle":"2021-06-12T16:30:07.318177Z","shell.execute_reply.started":"2021-06-12T16:30:07.304559Z","shell.execute_reply":"2021-06-12T16:30:07.317279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datasets_titles = [x.lower() for x in set(train_df['dataset_title'].unique()).union(set(train_df['dataset_label'].unique()))]","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:30:07.31981Z","iopub.execute_input":"2021-06-12T16:30:07.320293Z","iopub.status.idle":"2021-06-12T16:30:07.333128Z","shell.execute_reply.started":"2021-06-12T16:30:07.320257Z","shell.execute_reply":"2021-06-12T16:30:07.332289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor index in train_df['Id']:\n    publication_text = train_df[train_df['Id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append(' | '.join(label))\n\ntrain_df['final_label'] = labels","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:30:07.334612Z","iopub.execute_input":"2021-06-12T16:30:07.335109Z","iopub.status.idle":"2021-06-12T16:35:12.04188Z","shell.execute_reply.started":"2021-06-12T16:30:07.33507Z","shell.execute_reply":"2021-06-12T16:35:12.04097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['final_label'] = train_df['final_label'].apply(lambda x: tagger(x))\n#train_df['cleaned_label'] = train_df['cleaned_label'].apply(lambda x: tagger(x))\ntrain_df['cleaned_text'] = train_df['text'].apply(clean_text)","metadata":{"id":"HqhGf_v6lBbW","execution":{"iopub.status.busy":"2021-06-12T16:35:12.043351Z","iopub.execute_input":"2021-06-12T16:35:12.043753Z","iopub.status.idle":"2021-06-12T16:37:00.601618Z","shell.execute_reply.started":"2021-06-12T16:35:12.043713Z","shell.execute_reply":"2021-06-12T16:37:00.600766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_text'] = train_df['cleaned_text'].str[:max_len_text] # For OOM issues.","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.603017Z","iopub.execute_input":"2021-06-12T16:37:00.603376Z","iopub.status.idle":"2021-06-12T16:37:00.700224Z","shell.execute_reply.started":"2021-06-12T16:37:00.603341Z","shell.execute_reply":"2021-06-12T16:37:00.699355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.701667Z","iopub.execute_input":"2021-06-12T16:37:00.702006Z","iopub.status.idle":"2021-06-12T16:37:00.708449Z","shell.execute_reply.started":"2021-06-12T16:37:00.701971Z","shell.execute_reply":"2021-06-12T16:37:00.707411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.711897Z","iopub.execute_input":"2021-06-12T16:37:00.712409Z","iopub.status.idle":"2021-06-12T16:37:00.737271Z","shell.execute_reply.started":"2021-06-12T16:37:00.712303Z","shell.execute_reply":"2021-06-12T16:37:00.736314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.738814Z","iopub.execute_input":"2021-06-12T16:37:00.739158Z","iopub.status.idle":"2021-06-12T16:37:00.746222Z","shell.execute_reply.started":"2021-06-12T16:37:00.739124Z","shell.execute_reply":"2021-06-12T16:37:00.745346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_text']","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.747713Z","iopub.execute_input":"2021-06-12T16:37:00.748127Z","iopub.status.idle":"2021-06-12T16:37:00.75825Z","shell.execute_reply.started":"2021-06-12T16:37:00.748089Z","shell.execute_reply":"2021-06-12T16:37:00.757382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['final_label'].apply(lambda x: len(x.split(' '))).max()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.759589Z","iopub.execute_input":"2021-06-12T16:37:00.759988Z","iopub.status.idle":"2021-06-12T16:37:00.787362Z","shell.execute_reply.started":"2021-06-12T16:37:00.759953Z","shell.execute_reply":"2021-06-12T16:37:00.786558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['final_label']","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:37:00.788721Z","iopub.execute_input":"2021-06-12T16:37:00.789086Z","iopub.status.idle":"2021-06-12T16:37:00.798044Z","shell.execute_reply.started":"2021-06-12T16:37:00.789051Z","shell.execute_reply":"2021-06-12T16:37:00.797095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer \nfrom keras.preprocessing.sequence import pad_sequences\nfrom nltk.corpus import stopwords   \nfrom tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional, Attention\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.regularizers import l1_l2\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.nn import sparse_softmax_cross_entropy_with_logits\nimport tensorflow as tf","metadata":{"id":"ZLVyUbahkDbp","execution":{"iopub.status.busy":"2021-06-12T16:37:00.799448Z","iopub.execute_input":"2021-06-12T16:37:00.799855Z","iopub.status.idle":"2021-06-12T16:37:05.640859Z","shell.execute_reply.started":"2021-06-12T16:37:00.799822Z","shell.execute_reply":"2021-06-12T16:37:05.640011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BUFFER_SIZE = 20000\nBATCH_SIZE = 128\n\nencoder_maxlen = 512\ndecoder_maxlen = 125\nnum_words = 7000","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:40:16.045963Z","iopub.execute_input":"2021-06-12T16:40:16.046312Z","iopub.status.idle":"2021-06-12T16:40:16.051543Z","shell.execute_reply.started":"2021-06-12T16:40:16.04628Z","shell.execute_reply":"2021-06-12T16:40:16.050627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n# since < and > from default tokens cannot be removed\nfilters = '!\"#$%&()*+,-./:;=?@[\\\\]^_`{}~\\t\\n'\noov_token = '<unk>'\n\npaper_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words = num_words, oov_token=oov_token)\nlabel_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters=filters, oov_token=oov_token)\n\npaper_tokenizer.fit_on_texts(train_df['cleaned_text'])\nlabel_tokenizer.fit_on_texts(train_df['final_label'])\n\npaper_tokenizer.word_index = {e:i for e,i in paper_tokenizer.word_index.items() if i <= num_words} # <= because tokenizer is 1 indexed\nlabel_tokenizer.word_index = {e:i for e,i in label_tokenizer.word_index.items() if i <= num_words} # <= because tokenizer is 1 indexed\n\nencoder_vocab_size, decoder_vocab_size = len(paper_tokenizer.word_index), len(label_tokenizer.word_index)\n\nprint('encoder_vocab_size:', encoder_vocab_size)\nprint('decoder_vocab_size:', decoder_vocab_size)","metadata":{"id":"oshyy8joJ477","outputId":"c6b6bade-0837-4b68-cf05-2d7cebea9271","execution":{"iopub.status.busy":"2021-06-12T16:40:16.564945Z","iopub.execute_input":"2021-06-12T16:40:16.56526Z","iopub.status.idle":"2021-06-12T16:41:21.353536Z","shell.execute_reply.started":"2021-06-12T16:40:16.565231Z","shell.execute_reply":"2021-06-12T16:41:21.352582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_full = train_df.copy()\n#train_df = train_df_full.sample(1000)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:41:21.355092Z","iopub.execute_input":"2021-06-12T16:41:21.355457Z","iopub.status.idle":"2021-06-12T16:41:21.375762Z","shell.execute_reply.started":"2021-06-12T16:41:21.355419Z","shell.execute_reply":"2021-06-12T16:41:21.374773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for training\ninputs = paper_tokenizer.texts_to_sequences(train_df['cleaned_text'])\ntargets = label_tokenizer.texts_to_sequences(train_df['final_label'])\n\ninputs = tf.keras.preprocessing.sequence.pad_sequences(inputs, maxlen=encoder_maxlen, padding='post', truncating='post')\ntargets = tf.keras.preprocessing.sequence.pad_sequences(targets, maxlen=decoder_maxlen, padding='post', truncating='post')","metadata":{"id":"xQVyozHpKAov","execution":{"iopub.status.busy":"2021-06-12T16:41:21.377736Z","iopub.execute_input":"2021-06-12T16:41:21.37808Z","iopub.status.idle":"2021-06-12T16:42:06.442841Z","shell.execute_reply.started":"2021-06-12T16:41:21.378044Z","shell.execute_reply":"2021-06-12T16:42:06.441968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = tf.data.Dataset.from_tensor_slices((inputs, targets)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:06.444371Z","iopub.execute_input":"2021-06-12T16:42:06.444681Z","iopub.status.idle":"2021-06-12T16:42:08.122934Z","shell.execute_reply.started":"2021-06-12T16:42:06.444647Z","shell.execute_reply":"2021-06-12T16:42:08.122078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import regularizers\nl1 = regularizers.l1(0)\nlayernorm = 1e-3","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_angles(position, i, d_model):\n    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))\n    return position * angle_rates\n  \ndef positional_encoding(position, d_model):\n    angle_rads = get_angles(\n        np.arange(position)[:, np.newaxis],\n        np.arange(d_model)[np.newaxis, :],\n        d_model\n    )\n\n    # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n\n    # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n    pos_encoding = angle_rads[np.newaxis, ...]\n\n    return tf.cast(pos_encoding, dtype=tf.float32)\n\ndef create_padding_mask(seq):\n    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n    return seq[:, tf.newaxis, tf.newaxis, :]\n\n  \ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask\n\n\ndef scaled_dot_product_attention(q, k, v, mask):\n    matmul_qk = tf.matmul(q, k, transpose_b=True)\n\n    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n    if mask is not None:\n        scaled_attention_logits += (mask * -1e9)  \n\n    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n\n    output = tf.matmul(attention_weights, v)\n    return output, attention_weights\n\n\ndef point_wise_feed_forward_network(d_model, dff):\n    return tf.keras.Sequential([\n        tf.keras.layers.Dense(dff, activation='relu'),\n        tf.keras.layers.Dense(dff, activation='relu'),\n        tf.keras.layers.Dense(d_model)\n    ])\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n\n        assert d_model % self.num_heads == 0\n\n        self.depth = d_model // self.num_heads\n\n        self.wq = tf.keras.layers.Dense(d_model)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n\n        self.dense = tf.keras.layers.Dense(d_model)\n        \n    def split_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n    \n    def call(self, v, k, q, mask):\n        batch_size = tf.shape(q)[0]\n\n        q = self.wq(q)\n        k = self.wk(k)\n        v = self.wv(v)\n\n        q = self.split_heads(q, batch_size)\n        k = self.split_heads(k, batch_size)\n        v = self.split_heads(v, batch_size)\n\n        scaled_attention, attention_weights = scaled_dot_product_attention(\n            q, k, v, mask)\n\n        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n\n        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))\n        output = self.dense(concat_attention)\n            \n        return output, attention_weights\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(EncoderLayer, self).__init__()\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=layernorm)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=layernorm)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n    \n    def call(self, x, training, mask):\n        attn_output, _ = self.mha(x, x, x, mask)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)\n\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)\n\n        return out2\n\nclass DecoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(DecoderLayer, self).__init__()\n\n        self.mha1 = MultiHeadAttention(d_model, num_heads)\n        self.mha2 = MultiHeadAttention(d_model, num_heads)\n\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=layernorm)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=layernorm)\n        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=layernorm)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        self.dropout3 = tf.keras.layers.Dropout(rate)\n    \n    \n    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n        attn1 = self.dropout1(attn1, training=training)\n        out1 = self.layernorm1(attn1 + x)\n\n        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)\n        attn2 = self.dropout2(attn2, training=training)\n        out2 = self.layernorm2(attn2 + out1)\n\n        ffn_output = self.ffn(out2)\n        ffn_output = self.dropout3(ffn_output, training=training)\n        out3 = self.layernorm3(ffn_output + out2)\n\n        return out3, attn_weights_block1, attn_weights_block2\n\n\nclass Encoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, rate=0.1):\n        super(Encoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, self.d_model)\n\n        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n        \n    def call(self, x, training, mask):\n        seq_len = tf.shape(x)[1]\n\n        x = self.embedding(x)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n    \n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, training, mask)\n    \n        return x\n\nclass Decoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, rate=0.1):\n        super(Decoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n\n        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n\n        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n        self.dropout = tf.keras.layers.Dropout(rate)\n    \n    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n        seq_len = tf.shape(x)[1]\n        attention_weights = {}\n\n        x = self.embedding(x)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)\n\n        for i in range(self.num_layers):\n            x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)\n\n            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n    \n        return x, attention_weights\n\nclass Transformer(tf.keras.Model):\n    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1):\n        super(Transformer, self).__init__()\n\n        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate)\n\n        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate)\n\n        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n    \n    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask, dec_padding_mask):\n        enc_output = self.encoder(inp, training, enc_padding_mask)\n\n        dec_output, attention_weights = self.decoder(tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n\n        final_output = self.final_layer(dec_output)\n\n        return final_output, attention_weights","metadata":{"id":"vh3E7zb0JE_6","execution":{"iopub.status.busy":"2021-06-12T16:42:08.124285Z","iopub.execute_input":"2021-06-12T16:42:08.124609Z","iopub.status.idle":"2021-06-12T16:42:08.172251Z","shell.execute_reply.started":"2021-06-12T16:42:08.124574Z","shell.execute_reply":"2021-06-12T16:42:08.168007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyper-params\nnum_layers = 2\nd_model = 128\ndff = 128\nnum_heads = 2\nEPOCHS = 10","metadata":{"id":"qIcmtnhJO93-","execution":{"iopub.status.busy":"2021-06-12T16:42:08.174286Z","iopub.execute_input":"2021-06-12T16:42:08.174814Z","iopub.status.idle":"2021-06-12T16:42:08.180729Z","shell.execute_reply.started":"2021-06-12T16:42:08.174775Z","shell.execute_reply":"2021-06-12T16:42:08.179729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, d_model, warmup_steps=4000):\n        super(CustomSchedule, self).__init__()\n\n        self.d_model = d_model\n        self.d_model = tf.cast(self.d_model, tf.float32)\n\n        self.warmup_steps = warmup_steps\n    \n    def __call__(self, step):\n        arg1 = tf.math.rsqrt(step)\n        arg2 = step * (self.warmup_steps ** -1.5)\n\n        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:08.182103Z","iopub.execute_input":"2021-06-12T16:42:08.182675Z","iopub.status.idle":"2021-06-12T16:42:08.192138Z","shell.execute_reply.started":"2021-06-12T16:42:08.182639Z","shell.execute_reply":"2021-06-12T16:42:08.190807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate = CustomSchedule(d_model)\n#learning_rate = 0.0001\noptimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n\n\nloss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n\ndef loss_function(real, pred):\n    mask = tf.math.logical_not(tf.math.equal(real, 0))\n    loss_ = loss_object(real, pred)\n    #loss_ = sparse_softmax_cross_entropy_with_logits(real, pred)\n    mask = tf.cast(mask, dtype=loss_.dtype)\n    loss_ *= mask\n\n    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n\n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:08.194647Z","iopub.execute_input":"2021-06-12T16:42:08.195324Z","iopub.status.idle":"2021-06-12T16:42:08.503025Z","shell.execute_reply.started":"2021-06-12T16:42:08.195285Z","shell.execute_reply":"2021-06-12T16:42:08.502239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformer = Transformer(\n    num_layers, \n    d_model, \n    num_heads, \n    dff,\n    encoder_vocab_size + 1, \n    decoder_vocab_size + 1, \n    pe_input=encoder_vocab_size + 1, \n    pe_target=decoder_vocab_size + 1,\n    rate = 0.5\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:08.504405Z","iopub.execute_input":"2021-06-12T16:42:08.504771Z","iopub.status.idle":"2021-06-12T16:42:08.633816Z","shell.execute_reply.started":"2021-06-12T16:42:08.504736Z","shell.execute_reply":"2021-06-12T16:42:08.632996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_masks(inp, tar):\n    enc_padding_mask = create_padding_mask(inp)\n    dec_padding_mask = create_padding_mask(inp)\n\n    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n    dec_target_padding_mask = create_padding_mask(tar)\n    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n  \n    return enc_padding_mask, combined_mask, dec_padding_mask","metadata":{"id":"1wUH2rlpPVtd","execution":{"iopub.status.busy":"2021-06-12T16:42:08.63494Z","iopub.execute_input":"2021-06-12T16:42:08.63526Z","iopub.status.idle":"2021-06-12T16:42:08.639708Z","shell.execute_reply.started":"2021-06-12T16:42:08.635227Z","shell.execute_reply":"2021-06-12T16:42:08.638884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef train_step(inp, tar):\n    tar_inp = tar[:, :-1]\n    tar_real = tar[:, 1:]\n\n    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n\n    with tf.GradientTape() as tape:\n        predictions, _ = transformer(\n            inp, tar_inp, \n            True, \n            enc_padding_mask, \n            combined_mask, \n            dec_padding_mask\n        )\n\n        loss = loss_function(tar_real, predictions)\n        \n        #loss = jaccard_distance(tar_real, predictions)\n    gradients = tape.gradient(loss, transformer.trainable_variables)\n    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n\n    train_loss(loss, predictions)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:08.640987Z","iopub.execute_input":"2021-06-12T16:42:08.641669Z","iopub.status.idle":"2021-06-12T16:42:08.650139Z","shell.execute_reply.started":"2021-06-12T16:42:08.641591Z","shell.execute_reply":"2021-06-12T16:42:08.649254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_of_batches = len([x for x in enumerate(dataset)])\nfor epoch in range(EPOCHS):\n    start = time.time()\n\n    train_loss.reset_states()\n    \n    for (batch, (inp, tar)) in enumerate(dataset):\n        assert np.isnan(tar).any() == False\n        assert np.isnan(inp).any() == False\n        #print('N')\n        train_step(inp, tar)\n        assert np.isnan(train_loss.result()).any() == False, print(batch)\n            \n        if batch % 100 == 0:\n            print('Epoch {} Batch {} of {} Loss {:.4f}'.format(epoch + 1, batch, n_of_batches, train_loss.result()))\n\n    print('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result()))\n\n    print('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T16:42:08.651383Z","iopub.execute_input":"2021-06-12T16:42:08.651886Z","iopub.status.idle":"2021-06-12T16:58:05.436794Z","shell.execute_reply.started":"2021-06-12T16:42:08.651836Z","shell.execute_reply":"2021-06-12T16:58:05.434486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(input_document):\n    input_document = paper_tokenizer.texts_to_sequences([input_document])\n    input_document = tf.keras.preprocessing.sequence.pad_sequences(input_document, maxlen=encoder_maxlen, padding='post', truncating='post')\n\n    encoder_input = tf.expand_dims(input_document[0], 0)\n\n    decoder_input = [label_tokenizer.word_index['<bos>']]\n    output = tf.expand_dims(decoder_input, 0)\n    \n    for i in range(decoder_maxlen):\n        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(encoder_input, output)\n\n        predictions, attention_weights = transformer(\n            encoder_input, \n            output,\n            False,\n            enc_padding_mask,\n            combined_mask,\n            dec_padding_mask\n        )\n\n        predictions = predictions[: ,-1:, :]\n        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n\n        if predicted_id == label_tokenizer.word_index['<eos>']:\n            return tf.squeeze(output, axis=0), attention_weights\n\n        output = tf.concat([output, predicted_id], axis=-1)\n\n    return tf.squeeze(output, axis=0), attention_weights","metadata":{"id":"2m-m9dEaKER6","execution":{"iopub.status.busy":"2021-06-07T14:49:59.918939Z","iopub.status.idle":"2021-06-07T14:49:59.91963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract(input_document):\n    # not considering attention weights for now, can be used to plot attention heatmaps in the future\n    extract = evaluate(input_document=input_document)[0].numpy()\n    extract = np.expand_dims(extract[1:], 0)  # not printing <go> token\n    extracted = label_tokenizer.sequences_to_texts(extract)[0]  # since there is just one translated document\n    fix_separation = \"|\".join([x.strip() for x in extracted.split('|')])\n    return fix_separation","metadata":{"id":"yplngLf5szO4","execution":{"iopub.status.busy":"2021-06-06T19:25:35.910427Z","iopub.execute_input":"2021-06-06T19:25:35.910795Z","iopub.status.idle":"2021-06-06T19:25:35.917394Z","shell.execute_reply.started":"2021-06-06T19:25:35.910764Z","shell.execute_reply":"2021-06-06T19:25:35.916635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-06T19:25:36.255001Z","iopub.execute_input":"2021-06-06T19:25:36.255313Z","iopub.status.idle":"2021-06-06T19:25:36.262617Z","shell.execute_reply.started":"2021-06-06T19:25:36.255285Z","shell.execute_reply":"2021-06-06T19:25:36.261761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f05(tps, fps, fns):\n    up = 1.25*tps\n    down = (1.25*tps) + (0.25*fns) + fps\n    return up/down","metadata":{"execution":{"iopub.status.busy":"2021-06-06T19:25:36.656605Z","iopub.execute_input":"2021-06-06T19:25:36.656931Z","iopub.status.idle":"2021-06-06T19:25:36.662434Z","shell.execute_reply.started":"2021-06-06T19:25:36.656903Z","shell.execute_reply":"2021-06-06T19:25:36.661632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_tagger(text):\n    return \" \".join([x for x in text.split() if x not in ['<BOS>','<EOS>']])","metadata":{"execution":{"iopub.status.busy":"2021-06-06T19:25:36.905807Z","iopub.execute_input":"2021-06-06T19:25:36.906093Z","iopub.status.idle":"2021-06-06T19:25:36.910072Z","shell.execute_reply.started":"2021-06-06T19:25:36.906065Z","shell.execute_reply":"2021-06-06T19:25:36.909199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"sample_number = 50\nidx_sample_testing = random.sample(train_df.index.tolist(), sample_number)\ntesting_sample = train_df.loc[idx_sample_testing]\ntesting_sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T19:25:38.812558Z","iopub.execute_input":"2021-06-06T19:25:38.812939Z","iopub.status.idle":"2021-06-06T19:25:38.836441Z","shell.execute_reply.started":"2021-06-06T19:25:38.812908Z","shell.execute_reply":"2021-06-06T19:25:38.835722Z"}}},{"cell_type":"raw","source":"c = 0\nfor idx in idx_sample_testing:\n    c += 1\n    print(c, end = '\\r')\n    text_df = testing_sample.loc[idx]\n    prediction_string = extract(text_df.text)\n    testing_sample.loc[idx, 'prediction_string'] = prediction_string\n    cleaned_label = remove_tagger(testing_sample.loc[idx]['cleaned_label'])\n    print('Prediction:', prediction_string, '\\nLabel:', cleaned_label)\n    #print()\n    tp, fp, fn = 0, 0, 0\n    if prediction_string == '':\n        fn += 1\n    else:\n        #for stri in prediction_string.split('|'):\n        jaccard_score = jaccard(prediction_string, cleaned_label)\n        if jaccard_score >= 0.5:\n            tp += 1\n        else:\n            fp += 1\n\n    testing_sample.loc[idx, 'tp'] = tp\n    testing_sample.loc[idx, 'fp'] = fp\n    testing_sample.loc[idx, 'fn'] = fn\n    #gc.collect()\ntps, fps, fns = testing_sample[['tp','fp','fn']].sum().values\nf_score = f05(tps, fps, fns)\nprint('F05:', f_score)\n\n\n#results = list(zip(cuts, scores))","metadata":{"execution":{"iopub.status.busy":"2021-06-06T19:25:38.967Z","iopub.execute_input":"2021-06-06T19:25:38.96726Z","iopub.status.idle":"2021-06-06T19:26:54.526212Z","shell.execute_reply.started":"2021-06-06T19:25:38.967234Z","shell.execute_reply":"2021-06-06T19:26:54.52475Z"}}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-06-08T15:57:09.636948Z","iopub.execute_input":"2021-06-08T15:57:09.637817Z","iopub.status.idle":"2021-06-08T15:57:09.650525Z","shell.execute_reply.started":"2021-06-08T15:57:09.637684Z","shell.execute_reply":"2021-06-08T15:57:09.649495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"id":"WYbZIguWZdxC","execution":{"iopub.status.busy":"2021-06-08T15:57:09.652647Z","iopub.execute_input":"2021-06-08T15:57:09.653199Z","iopub.status.idle":"2021-06-08T15:57:09.673593Z","shell.execute_reply.started":"2021-06-08T15:57:09.653154Z","shell.execute_reply":"2021-06-08T15:57:09.672366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"id":"UbNMOYO2ZmSS","outputId":"8d54cf6b-2ddb-42d4-e2cf-dc193888281e","execution":{"iopub.status.busy":"2021-06-08T15:57:30.573979Z","iopub.execute_input":"2021-06-08T15:57:30.574317Z","iopub.status.idle":"2021-06-08T15:57:30.600819Z","shell.execute_reply.started":"2021-06-08T15:57:30.574289Z","shell.execute_reply":"2021-06-08T15:57:30.599215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].apply(lambda x: read_append_return(x, test_files_path))","metadata":{"id":"DOEDCaGiaB51","execution":{"iopub.status.busy":"2021-06-08T15:58:03.444772Z","iopub.execute_input":"2021-06-08T15:58:03.445186Z","iopub.status.idle":"2021-06-08T15:58:03.502233Z","shell.execute_reply.started":"2021-06-08T15:58:03.445155Z","shell.execute_reply":"2021-06-08T15:58:03.501244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['PredictionString'] = sample_sub['text'].apply(extract)","metadata":{"id":"r2E6IKi_aXHx","execution":{"iopub.status.busy":"2021-06-08T16:01:38.318923Z","iopub.execute_input":"2021-06-08T16:01:38.31931Z","iopub.status.idle":"2021-06-08T16:01:38.3381Z","shell.execute_reply.started":"2021-06-08T16:01:38.31928Z","shell.execute_reply":"2021-06-08T16:01:38.335799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T16:01:38.488244Z","iopub.execute_input":"2021-06-08T16:01:38.488636Z","iopub.status.idle":"2021-06-08T16:01:38.502566Z","shell.execute_reply.started":"2021-06-08T16:01:38.488596Z","shell.execute_reply":"2021-06-08T16:01:38.501089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = sample_sub.drop('text', axis = 1)","metadata":{"id":"dN37nhahaS_W","outputId":"0b9bb060-9537-448b-8a58-06e292e68191","execution":{"iopub.status.busy":"2021-06-06T18:00:27.36932Z","iopub.status.idle":"2021-06-06T18:00:27.369942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T18:00:27.371014Z","iopub.status.idle":"2021-06-06T18:00:27.371606Z"},"trusted":true},"execution_count":null,"outputs":[]}]}