{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Sentence-level analysis using Transformer model in Keras\n> Baseline from Poe Dator. See [here](https://www.kaggle.com/poedator/sentence-level-analysis-with-transformer/)","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport pickle\nfrom collections import defaultdict, Counter\nimport gc\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, plot_confusion_matrix\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\n%matplotlib inline\nfrom IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\")) # full screen width of Jupyter notebook\npd.options.display.max_rows, pd.options.display.max_columns = 500, 100\n\n# NLP imports\nimport nltk\n\n# Neural network imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nprint( 'tf version:', tf.__version__)\n\nos.environ['PYTHONHASHSEED']=str(123)\ntf.random.set_seed(123)\nnp.random.seed(123)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\" Loading data\"\"\"\ndata_path = '../input/coleridgeinitiative-show-us-the-data/'\n\ndef read_json_from_folder(folder_name):\n    json_dict = {}\n    for filename in os.listdir(folder_name):\n        with open(os.path.join(folder_name, filename)) as f:\n            json_dict[filename[:-5]] = json.load(f)\n    return json_dict\n\n# train_dict = read_json_from_folder(os.path.join(data_path, 'train'))\n# test_dict = read_json_from_folder(os.path.join(data_path, 'test'))\n# train_df = pd.read_csv(os.path.join(data_path, 'train.csv'))\ntrain_df = pd.read_csv('../input/ci-train-with-ext-data/train_df.csv')\nsample_sub = pd.read_csv(os.path.join(data_path,'sample_submission.csv'))\n    \n# len(train_dict), len(test_dict), \ntrain_df.shape, sample_sub.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ext_cleaned_label'] = train_df['ext_cleaned_label'].apply(lambda x: x.split('|'))\ntrain_df.drop_duplicates(subset='Id', keep='last', inplace=True)\nprint(train_df.shape)\ntrain_df.sample(5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub_df = train_df.set_index('Id')[['pub_title', 'dataset_title', 'ext_cleaned_label']]\npub_df['n_refs'] = pub_df['ext_cleaned_label'].apply(lambda x: len(x)).astype(int)\npub_df = pub_df.rename(columns={'ext_cleaned_label':'refs'})\npub_df.head()\n\n\"\"\"Adding publications' texts from loaded json files\"\"\"\npub_df['raw_text'] = pd.Series(read_json_from_folder(os.path.join(data_path, 'train')))\npub_df['n_sections'] = pub_df['raw_text'].apply(lambda x: len(x)).astype(int)\n\n\"\"\" decoding raw text to simple text \"\"\"\npub_df['text'] = pub_df['raw_text'].apply(\n    lambda x:'\\n'.join([z for y in x for z in y.values()]))\npub_df['pub_len'] = pub_df.text.str.len()\n\nprint (pub_df.shape)\n\npub_df.sample(5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del pub_df['raw_text']\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\"\"\" classify train set sentences that contain references to datasets \"\"\"\nsentences = []\n\nfor row in tqdm(pub_df.itertuples(), total = pub_df.shape[0]):\n    for sent in nltk.sent_tokenize(re.sub(r'\\.?\\n', '. ', row.text)):\n        found_flag = False\n        for r in row.refs:\n            if r in sent.lower():\n                sentences.append(\n                    {'Id':row.Index, 'sentence':sent, 'ref':r, 'group':'train_1'})\n                found_flag = True\n        # save empty sencences,\n        if not found_flag:\n            if np.random.rand(1)[0] > 0.0:\n                sentences.append(\n                    {'Id':row.Index, 'sentence':sent,'group':'train_0'})\n\nprint(\"Num of sentences collected:\", len(sentences))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\" create DF with sentences \"\"\"\n\"\"\" group feature indicates:\n        - sentences with dataset titles ('train_1'),\n        - sentences witout dataset titles ('train_1'),\n        - sentences from publications without matches ('val') \"\"\"\nsent_df = pd.DataFrame(sentences)\n# sent_df = sent_df.append(pd.DataFrame(sentences_empty)) # append empty sentences\nsent_df = sent_df.reset_index(drop=True)  # reset after append\nsent_df['Id'] = sent_df['Id'].astype('category')\nsent_df['ref'] = sent_df['ref'].astype('category')\nsent_df['group'] = sent_df['group'].astype('category')\nsent_df['n_chars'] = sent_df.sentence.str.len()\n\n# drop short empty sences (either chapter titles or tables components)\nsent_df = sent_df.drop(sent_df[(sent_df.n_chars < 40) & (sent_df.ref.isna())].index)\nprint (sent_df.shape)\n\nsent_df['group'].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2 random and 2 positive examples\nsent_df.sample(2).append(sent_df[sent_df.group == 'train_1'].sample(2))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" DATASET CREATION\nDue to the abundance of the negative examples, we limit their number to 3x the number of positive examples\nThis also saves time in model performance.\nNB: Consider benefits of using all negative examples\n\"\"\"\nneg_multiple = 9  # multiplier to get number of negative examples\nstopword_list = nltk.corpus.stopwords.words('english')\n\ndf = sent_df[sent_df.group == 'train_1']  # positive examples\ndf = df.append(sent_df[sent_df.group == 'train_0'].sample(df.shape[0] * neg_multiple))\n# df = df.drop(columns = ['n_chars'])  # keep 'Id', \ndf['clean'] = df.sentence.str.lower().replace(r\"[^a-z ]+\",\"\", regex=True)\ndf['n_words'] = df.clean.apply(lambda x: len(x.split()))\nprint(df.shape)\n\ndf['group'].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the referenced documents by ID\n# This ensures that references from same publication are not present in both train and val sets\n\nid_train, id_val = train_test_split(df.Id.unique(), test_size=0.1, random_state=42)\ntrain_idx = df.reset_index()[df.Id.isin(id_train).values].index\nval_idx = df.reset_index()[df.Id.isin(id_val).values].index","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# While there are very long sentences (split defects?), most are under 50 0chars long \nprint (\"max number of characters in sentence:\", df.n_chars.max())\ndf[df.n_chars < 1000].n_chars.hist(bins=20);\ndf.sample(3)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize the sentences\n%time df['tokenized'] = df.clean.progress_apply(lambda x: [ \\\n    w for w in nltk.word_tokenize(x[:500]) if w not in stopword_list])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_words = Counter()\nfor words in tqdm(df.tokenized.values):\n    unique_words.update(words)\nprint (f\"Unique words: {len(unique_words)}\")    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" assess opportunity to reduce vocab \"\"\"\n# count rare words\nprint(\"Percent of words in corpus by num of occurences\")\nprint(pd.Series(unique_words.values()).value_counts().head(10)/len(unique_words), '\\n')\n\n# count words by length\nprint(\"Percent of words in corpus by length\")\nprint(pd.Series(unique_words.keys(), name=\"words\").str.len().value_counts().to_frame().reset_index().\\\n    sort_values(by='index').head(10).set_index('index')/len(unique_words))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove short and infrequent words\nmin_occurencies = 10\nmin_word_len = 3\nmy_vocab = {k:v for k, v in unique_words.items() if v>=min_occurencies and len(k)>= min_word_len}\nmy_vocab = {k: v for k, v in sorted(my_vocab.items(), key=lambda item: item[1], reverse=True)}\nvocab_size = len(my_vocab)\nprint (f\"Words to be used for regression: {vocab_size}\") ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" preparing feed for NN models\"\"\"\n\nnum_classes = 2\n\nfilters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\'\\n' + '0123456789'\ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=25000, lower=True, \n                                                  filters=filters, oov_token='<OOV>')\n\n# tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=len(list(my_vocab)))\ntokenizer.fit_on_texts(my_vocab.keys())\n# X_tokenized = tokenizer.texts_to_sequences(df.clean)\n%time df['tokens_tf'] = tokenizer.texts_to_sequences(df.clean)\nprint (f\"Input sentences tokenized with {tokenizer.get_config()['num_words']} words vocab\")\ny = ~df.ref.isna()\n\nmaxlen = 500\nlen_max = df.clean.str.len().max()\nprint (f\"Max cleaned title length: {len_max}; limiting/padding sentences to {maxlen} words\")\n\nX_padded = tf.keras.preprocessing.sequence.pad_sequences(\n    df.tokens_tf, maxlen=maxlen, padding='pre',)\n\nX_train = X_padded[train_idx,:]\nX_val = X_padded[val_idx,:]\ny = 1 - df.ref.isna().astype(int)\ny_train = y.iloc[train_idx]\ny_val = y.iloc[val_idx]\nprint(\"Subsets shapes: \", X_train.shape, X_val.shape, y_train.shape, y_val.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" build transformer model\"\"\"\n\nembed_dim = 32  # Embedding size for each token\nnum_heads = 2  # Number of attention heads\nff_dim = 32  # Hidden layer size in feed forward network inside transformer\n\nclass TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n\nclass TokenAndPositionEmbedding(layers.Layer):\n    def __init__(self, maxlen, vocab_size, embed_dim):\n        super(TokenAndPositionEmbedding, self).__init__()\n        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        x = self.token_emb(x)\n        return x + positions\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(2, activation=\"softmax\")(x)\n\nmodel_t = keras.Model(inputs=inputs, outputs=outputs)\n# model_t.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_t.compile(loss='sparse_categorical_crossentropy',\n              optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy'])\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0,\n                patience=0, verbose=1, mode='auto', baseline=None, restore_best_weights=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_t = model_t.fit(X_train, y_train, \n                  validation_data=(X_val, y_val),\n                  epochs=1, batch_size=32, verbose=1, callbacks=callback)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_t.history.values()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# save model weights\nmodel_t.save_weights('./model/sent_transformer')\n\n# Vocab and tf tokenizer\nwith open('tokenizer.pickle', 'wb') as handle:\n    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)   \nwith open('my_vocab.pickle', 'wb') as handle:\n    pickle.dump(my_vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n# Dataframe with all selected sentences\nsent_df.to_pickle('sent_df.pickle')\npub_df.to_pickle('pub_df.pickle')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls * -lh","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sent_df\ndel pub_df\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import matching module\nfrom fuzzywuzzy import fuzz\n\n# prepare list of dataset titles to match\nds_titles = np.unique(np.concatenate(train_df['ext_cleaned_label'].values))\nds_titles.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_path = '../input/coleridgeinitiative-show-us-the-data/test'\ntest_df = sample_sub.Id.to_frame().set_index('Id')\ntest_sentences = {}\ncandidate_threshold = 0.3\nacceptance_score = 80\n\ndef read_json_pub(Id):\n    filename = os.path.join(test_data_path, Id+'.json')\n    with open(filename) as f:\n        json_pub = json.load(f)\n    return json_pub\n\nfor row in tqdm(test_df.itertuples(), total = test_df.shape[0]):\n#     Load text\n    raw_text = read_json_pub(row.Index)\n    text = '\\n'.join([z for y in raw_text for z in y.values()])\n\n#     split and clean sentences\n    sentences = nltk.sent_tokenize(re.sub(r'\\.?\\n', '. ', text))\n    sentences = [re.sub(r\"[^a-z ]+\",\"\", s.lower()) for s in sentences]\n    \n# tokenize\n    tokens = tokenizer.texts_to_sequences(sentences)\n    tokens = tf.keras.preprocessing.sequence.pad_sequences(\n        tokens, maxlen=maxlen, padding='pre',)\n\n# Predict candidates sentences that may contain DS references\n    y_pred = model_t.predict(tokens, batch_size=32)\n    sent_candidates = np.array(sentences)[y_pred[:,1] > candidate_threshold]\n    test_sentences[row.Index] = sent_candidates\n\n#  process candidate sentences for given pub\n    ds_candidates = set()\n    for sent in sent_candidates:\n        scores = [fuzz.partial_ratio(sent, title) for title in ds_titles]\n        best_fit_title_index = np.argmax(scores)\n        if max(scores) > acceptance_score:\n            ds_candidates.add(ds_titles[np.argmax(scores)])\n    prediction_string = ' | '.join(ds_candidates)\n#     print (prediction_string)\n    test_df.loc[row.Index, 'PredictionString'] = prediction_string","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[['PredictionString']].to_csv('submission.csv')","metadata":{},"execution_count":null,"outputs":[]}]}