{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 style='background:teal; color:white; padding:20px;'>\nColeridge: Sentence analysis</h1>\n\n\n**Identifying sentences containinig dataset titles**\n\nversion history:\n- v.1-2 sentence split, Naive NBayes model baseline\n- v.3 - added Transformer model for sentence identification\n- v.4,5 - code cleanup. Added prediction for test set with simple string matching.\n- v.6 - consecutive processing of test pubs to save RAM\n- v.7 - removed Naive Bayes model, improved positive match detection","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport pickle\nfrom collections import defaultdict, Counter\nimport gc\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, plot_confusion_matrix\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\n%matplotlib inline\nfrom IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\")) # full screen width of Jupyter notebook\npd.options.display.max_rows, pd.options.display.max_columns = 500, 100\n\n# NLP imports\nimport nltk\n\n# Neural network imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nprint( 'tf version:', tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import multiprocessing as mp\nimport pandas.util.testing as pdt\nmp.cpu_count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\" Loading data\"\"\"\ndata_path = '../input/coleridgeinitiative-show-us-the-data/'\n\ndef read_json_from_folder(folder_name):\n    json_dict = {}\n    for filename in os.listdir(folder_name):\n        with open(os.path.join(folder_name, filename)) as f:\n            json_dict[filename[:-5]] = json.load(f)\n    return json_dict\n\n# train_dict = read_json_from_folder(os.path.join(data_path, 'train'))\n# test_dict = read_json_from_folder(os.path.join(data_path, 'test'))\ntrain_df = pd.read_csv(os.path.join(data_path, 'train.csv'))\nsample_sub = pd.read_csv(os.path.join(data_path,'sample_submission.csv'))\n    \n# len(train_dict), len(test_dict), \ntrain_df.shape, sample_sub.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Publications Dataframe creation `pubs_df`","metadata":{}},{"cell_type":"code","source":"\"\"\" build PUB_DF by repacking data from train_df \"\"\"\npub_df = train_df.groupby('Id').agg({'pub_title':'first', 'dataset_label':'nunique',\n                                    'cleaned_label':lambda x: sorted(x)})\npub_df = pub_df.rename(columns = {'dataset_label':'n_refs', 'cleaned_label':'refs'})\n# note that `refs` now contains sorted ground truth references","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"Adding publications' texts from loaded json files\"\"\"\npub_df['raw_text'] = pd.Series(read_json_from_folder(os.path.join(data_path, 'train')))\npub_df['n_sections'] = pub_df['raw_text'].apply(lambda x: len(x)).astype(int)\n\n\"\"\" decoding raw text to simple text \"\"\"\npub_df['text'] = pub_df['raw_text'].apply(\n    lambda x:'\\n'.join([z for y in x for z in y.values()]))\npub_df['pub_len'] = pub_df.text.str.len()\n\nprint (pub_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" find and count exact text matches in train_df and pub_df \"\"\"\nfor row in tqdm(train_df.itertuples(), total = train_df.shape[0]):\n    text = pub_df.loc[row.Id, 'text'].lower()\n#     print (i, len(text), row.dataset_title in text)\n    train_df.loc[row.Index, 'found'] = row.dataset_label.lower() in text  # same result with lower\ntrain_df.found = train_df.found.astype(int)\nprint (f\"Confirmed {(train_df.found != 0).sum()} matches.\")\nprint (f\"Failed to confirm {(train_df.found != 1).sum()} matches. Examples: {train_df[train_df.found == 0].index[:5].values.tolist()}\")\n\n# Add number of datasets, whose names were found in each publication\npub_df['ref_matched'] = train_df.groupby('Id').agg({'found':'sum'})\n\npub_df.sample(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"del pub_df['raw_text']\ngc.collect()","metadata":{}},{"cell_type":"code","source":"pub_df.sample(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sentence dataframe creation","metadata":{}},{"cell_type":"code","source":"# prepare list of dataset titles to match\nds_titles = set().union(set(train_df.cleaned_label.unique()),\n                        set(train_df.dataset_label.str.lower().unique()),\n                        set(train_df.dataset_title.str.lower().unique()),  )\nds_titles = np.array(list(ds_titles))\nds_titles.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = train_df.groupby('dataset_title').agg({'dataset_label':'unique', 'cleaned_label':'unique'}).reset_index()\n\nfor row in dataset_df.itertuples():\n    labels = [*row.dataset_label.tolist(), *row.cleaned_label.tolist()]\n    labels = set([lab.lower() for lab in labels])\n    dataset_df.loc[row.Index, 'labels'] = '|'.join(labels)\n\ndataset_df.labels = dataset_df.labels.apply(lambda x: x.split('|'))\ndataset_df.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\" classify train set sentences that contain references to datasets \"\"\"\n# n = 1000\nsentences = []\nfor row in tqdm(pub_df.itertuples(), total = pub_df.shape[0]):\n    clean_text = re.sub(r\"[^a-z \\.\\n]+\",\"\", row.text.lower())\n    titles_shortlist = [t for t in ds_titles if t in clean_text]  # shortlist saves time\n\n#     capture similar ds title variants\n    expanded_refs = []\n    for title in row.refs:\n        for ds_row in dataset_df.itertuples():\n            if title in ds_row.labels:\n                expanded_refs.extend(ds_row.labels)\n        expanded_refs = list(set(row.refs + expanded_refs))\n    \n    sentences_list = nltk.sent_tokenize(re.sub(r'\\.?\\n', '. ', row.text))\n    sentences_list = [re.sub(r\"[^a-z \\.]+\",\"\", s.lower()) for s in sentences_list]\n    for sent in sentences_list:\n        found_flag = False\n        for title in titles_shortlist:\n            if title in sent:\n                found_flag = True\n                group = 'TP' if title in expanded_refs else 'FP'\n                sentences.append({'Id':row.Index,'sentence':sent,'match':title,'group':group})\n        if not found_flag:\n            group = 'N' if row.n_refs == row.ref_matched else 'UNK'  \n            sentences.append({'Id':row.Index,'sentence':sent,'match':None,'group':group})\n\nsent_df = pd.DataFrame(sentences)\nprint(sent_df.shape)\nsent_df.group.value_counts()\n# was 9631 FPs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sent_df['n_chars'] = sent_df.sentence.str.len()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sent_df[sent_df.group=='FP'].sample(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample false positive\nid = '84f2ca4e-2d1f-40f5-857b-7d41fe724645'\ntext = pub_df.loc[id, 'text']\nclean_text = re.sub(r\"[^a-z \\n\\.]+\",\"\", text.lower())\npub_df.loc[id, 'refs']\n# mm in ds_titles","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2 random and 2 positive examples\nsent_df.sample(2).append(sent_df[sent_df.group == 'TP'].sample(2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELS FOR PREDICTION OF SENTENCES WITH DS TITLES","metadata":{}},{"cell_type":"markdown","source":"### Dataset creation","metadata":{}},{"cell_type":"code","source":"\"\"\" DATASET CREATION\nDue to the abundance of the negative examples, we limit their number to 3x the number of positive examples\nThis also saves time in model performance.\nNB: Consider benefits of using all negative examples\n\"\"\"\nneg_multiple = 9  # multiplier to get number of negative examples\nstopword_list = nltk.corpus.stopwords.words('english')\n\ndf = sent_df[sent_df.group == 'TP']  # positive examples\ndf = df.append(sent_df[sent_df.group == 'N'].sample(df.shape[0] * neg_multiple))\n# df = df.drop(columns = ['n_chars'])  # keep 'Id', \ndf['clean'] = df.sentence.str.lower().replace(r\"[^a-z ]+\",\"\", regex=True)\ndf['n_words'] = df.clean.apply(lambda x: len(x.split()))\ndf.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the referenced documents by ID\n# This ensures that references from same publication are not present in both train and val sets\n\nid_train, id_val = train_test_split(df.Id.unique(), test_size=0.1, random_state=42)\ntrain_idx = df.reset_index()[df.Id.isin(id_train).values].index\nval_idx = df.reset_index()[df.Id.isin(id_val).values].index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# While there are very long sentences (split defects?), most are under 50 0chars long \nprint (\"max number of characters in sentence:\", df.n_chars.max())\ndf[df.n_chars < 1000].n_chars.hist(bins=20);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build vocab","metadata":{}},{"cell_type":"code","source":"# Tokenize the sentences\n%time df['tokenized'] = df.clean.progress_apply(lambda x: [ \\\n    w for w in nltk.word_tokenize(x[:500]) if w not in stopword_list])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_words = Counter()\nfor words in tqdm(df.tokenized.values):\n    unique_words.update(words)\nprint (f\"Unique words: {len(unique_words)}\")    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" assess opportunity to reduce vocab \"\"\"\n# count rare words\nprint(\"Percent of words in corpus by num of occurences\")\nprint(pd.Series(unique_words.values()).value_counts().head(10)/len(unique_words), '\\n')\n\n# count words by length\nprint(\"Percent of words in corpus by length\")\nprint(pd.Series(unique_words.keys(), name=\"words\").str.len().value_counts().to_frame().reset_index().\\\n    sort_values(by='index').head(10).set_index('index')/len(unique_words))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove short and infrequent words\nmin_occurencies = 10\nmin_word_len = 3\nmy_vocab = {k:v for k, v in unique_words.items() if v>=min_occurencies and len(k)>= min_word_len}\nmy_vocab = {k: v for k, v in sorted(my_vocab.items(), key=lambda item: item[1], reverse=True)}\nvocab_size = len(my_vocab)\nprint (f\"Words to be used for regression: {vocab_size}\") ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Naïve Bayes model","metadata":{}},{"cell_type":"code","source":"# create data for training and validation\nnb_X_train = df.iloc[train_idx].tokenized\nnb_X_val = df.iloc[val_idx].tokenized\n\nnb_y_train = (df[(df.Id.isin(id_train))].group=='TP').astype('int')\nnb_y_val = (df[(df.Id.isin(id_val))].group=='TP').astype('int')\n\n# prepare the vectorizer\nvectorizer = CountVectorizer(vocabulary = list(my_vocab.keys()))\n%time vectorizer = vectorizer.fit(df.tokenized.apply(lambda x: ' '.join(x)).values)\n\n# vectorize\nnb_X_train = vectorizer.transform(nb_X_train.apply(lambda x: ' '.join(x)).values)\nnb_X_val = vectorizer.transform(nb_X_val.apply(lambda x: ' '.join(x)).values)\n\nprint(nb_X_train.shape, nb_X_val.shape, nb_y_train.shape, nb_y_val.shape)\nnb_X_train.shape, nb_X_val.shape, nb_y_train.shape, nb_y_val.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run Naïve Bayes model\nnb_model = MultinomialNB().fit(nb_X_train, nb_y_train)\nnb_y_pred = nb_model.predict(nb_X_val)\nnb_score = accuracy_score(nb_y_val, nb_y_pred)\nprint (f\"Naïve Bayes baseline accuracy score: {nb_score:.4}\")\n\nplot_confusion_matrix (nb_model, nb_X_val, nb_y_val);\nplt.title(f\"Confusion Matrix. Accuracy = {nb_score*100:.2f}\", fontsize=16);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer model\nLSTM would be faster but shows somewhat worse performance","metadata":{}},{"cell_type":"code","source":"\"\"\" preparing feed for NN models\"\"\"\n\nnum_classes = 2\n\nfilters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\'\\n' + '0123456789'\ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=25000, lower=True, \n                                                  filters=filters, oov_token='<OOV>')\n\n# tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=len(list(my_vocab)))\ntokenizer.fit_on_texts(my_vocab.keys())\n# X_tokenized = tokenizer.texts_to_sequences(df.clean)\n%time df['tokens_tf'] = tokenizer.texts_to_sequences(df.clean)\nprint (f\"Input sentences tokenized with {tokenizer.get_config()['num_words']} words vocab\")\n# y = df.group == 'TP'\n\nmaxlen = 500\nlen_max = df.clean.str.len().max()\nprint (f\"Max cleaned title length: {len_max}; limiting/padding sentences to {maxlen} words\")\n\nX_padded = tf.keras.preprocessing.sequence.pad_sequences(\n    df.tokens_tf, maxlen=maxlen, padding='pre',)\n\nX_train = X_padded[train_idx,:]\nX_val = X_padded[val_idx,:]\ny =(df.group == 'TP').astype(int)\ny_train = y.iloc[train_idx]\ny_val = y.iloc[val_idx]\nprint(\"Subsets shapes: \", X_train.shape, X_val.shape, y_train.shape, y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" build transformer model\"\"\"\n\nembed_dim = 32  # Embedding size for each token\nnum_heads = 2  # Number of attention heads\nff_dim = 32  # Hidden layer size in feed forward network inside transformer\n\nclass TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n\nclass TokenAndPositionEmbedding(layers.Layer):\n    def __init__(self, maxlen, vocab_size, embed_dim):\n        super(TokenAndPositionEmbedding, self).__init__()\n        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        x = self.token_emb(x)\n        return x + positions\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(2, activation=\"softmax\")(x)\n\nmodel_t = keras.Model(inputs=inputs, outputs=outputs)\n# model_t.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_t.compile(loss='sparse_categorical_crossentropy',\n              optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy'])\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0,\n                patience=0, verbose=1, mode='auto', baseline=None, restore_best_weights=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model will be trained for 1 epoch only  - this already gives close to 99% accuracy and saves time. 2d epoch will overfit already.\n\nThere may be some potential in hyperparameters optimization. \n\nGPU accelerates training by factor of over 10.","metadata":{}},{"cell_type":"code","source":"history_t = model_t.fit(X_train, y_train, \n                  validation_data=(X_val, y_val),\n                  epochs=1, batch_size=32, verbose=1, callbacks=callback)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_t.history.values()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analyze pubs with 0 direct matches\n\na few publications were not with the dataset names by string search. Let's see if our models can point to the relevant sentences which contain dataset names.","metadata":{}},{"cell_type":"code","source":"# preparing dataset with pubs with zero refenrece matches\npubs_0 = pub_df[pub_df.ref_matched == 0]\nprint(f\"There are {pubs_0.shape[0]} pubs with zero dataset title matches\")\npubs_0.sample(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"Preparing dataset with sentences \"\"\"\nsent_0 = sent_df[sent_df.group=='UNK']\nsent_0 = sent_0.drop(sent_0[(sent_0.n_chars < 40)].index)  # drop too short sentences\nsent_0 = sent_0.drop(sent_0[(sent_0.n_chars > 500)].index)  # drop too long sentences\nsent_0['clean'] = sent_0.sentence.str.lower().replace(r\"[^a-z ]+\",\"\", regex=True)\nprint(sent_0.shape)\nsent_0.sample(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test with Naïve Bayes model","metadata":{}},{"cell_type":"code","source":"%time sent_0['tokenized'] = sent_0.clean.progress_apply(lambda x: [ \\\n    w for w in nltk.word_tokenize(x) if w not in stopword_list])\n\n%time sent_0_X = vectorizer.transform(sent_0.tokenized.apply(lambda x: ' '.join(x)).values)\nsent_0['pred_nb'] = nb_model.predict(sent_0_X).astype(int)\n\n# count sentences with possible title\nprint (f\"candidates found with Naïve Bayes: {sent_0.pred_nb.sum()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test with transformers model","metadata":{}},{"cell_type":"code","source":"sent_0['tokens_tf'] = tokenizer.texts_to_sequences(sent_0.clean)\nX_padded_0 = tf.keras.preprocessing.sequence.pad_sequences(\n    sent_0.tokens_tf, maxlen=maxlen, padding='pre',)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nsent_0['tokens_tf'] = tokenizer.texts_to_sequences(sent_0.clean)\nX_padded_0 = tf.keras.preprocessing.sequence.pad_sequences(\n    sent_0.tokens_tf, maxlen=maxlen, padding='pre',)\ny_pred_logits = model_t.predict(X_padded_0, batch_size=32)\ny_pred = y_pred_logits.argmax(axis=1)\nsent_0['pred_tf'] = y_pred\nprint (f\"identified {y_pred.sum()} candidate sentences with possible dataset titles\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pubs_0.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" show selected sentences \"\"\"\nfor row in sent_0[sent_0.pred_tf==1].sample(5).itertuples():\n    print(\"ID:  \", row.Id)\n    print(\"Sentence:  \", row.sentence)\n    print(\"Ground truth:  \", pub_df.loc[row.Id, 'refs'])\n    print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there is  number of sentences with logit values between 0.1 and 0.5, \nprint(\"Num candidates with logit > 0.5: \",  y_pred_logits[y_pred_logits[:,1]>0.5].shape[0])\nprint(\"Num candidates with logit > 0.1: \",  y_pred_logits[y_pred_logits[:,1]>0.1].shape[0])\nsns.histplot(y_pred_logits[:,1][y_pred_logits[:,1]>0.1], bins = 9);\nsns.histplot(y_pred_logits[:,1][y_pred_logits[:,1]>0.5], bins = 5, color='g', alpha = 1);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving interim results","metadata":{}},{"cell_type":"code","source":"%%time\n# save model weights\nmodel_t.save_weights('./model/sent_transformer')\n\n# Vocab and tf tokenizer\nwith open('tokenizer.pickle', 'wb') as handle:\n    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)   \nwith open('my_vocab.pickle', 'wb') as handle:\n    pickle.dump(my_vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n# # collected sentences\n# with open('sentences.pickle', 'wb') as handle:\n#     pickle.dump(sentences, handle, protocol=pickle.HIGHEST_PROTOCOL)   \n# len(sentences)\n\n# Dataframe with all selected sentences\nsent_df.to_pickle('sent_df.pickle')\npub_df.to_pickle('pub_df.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls * -lh","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# predict test for submission","metadata":{}},{"cell_type":"code","source":"del sent_df\ndel pub_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import matching module\nfrom fuzzywuzzy import fuzz\n\n# prepare list of dataset titles to match\nds_titles = train_df.cleaned_label.unique()\nds_titles.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_path = '../input/coleridgeinitiative-show-us-the-data/test'\ntest_df = sample_sub.Id.to_frame().set_index('Id')\ntest_sentences = {}\ncandidate_threshold = 0.3\nacceptance_score = 80\n\ndef read_json_pub(Id):\n    filename = os.path.join(test_data_path, Id+'.json')\n    with open(filename) as f:\n        json_pub = json.load(f)\n    return json_pub\n\nfor row in tqdm(test_df.itertuples(), total = test_df.shape[0]):\n#     Load text\n    raw_text = read_json_pub(row.Index)\n    text = '\\n'.join([z for y in raw_text for z in y.values()])\n\n#     split and clean sentences\n    sentences = nltk.sent_tokenize(re.sub(r'\\.?\\n', '. ', text))\n    sentences = [re.sub(r\"[^a-z ]+\",\"\", s.lower()) for s in sentences]\n    \n# tokenize\n    tokens = tokenizer.texts_to_sequences(sentences)\n    tokens = tf.keras.preprocessing.sequence.pad_sequences(\n        tokens, maxlen=maxlen, padding='pre',)\n\n# Predict candidates sentences that may contain DS references\n    y_pred = model_t.predict(tokens, batch_size=32)\n    sent_candidates = np.array(sentences)[y_pred[:,1] > candidate_threshold]\n    test_sentences[row.Index] = sent_candidates\n#     print (row.Index, len(candidates))\n    \n#  process candidate sentences for given pub\n    ds_candidates = set()\n    for sent in sent_candidates:\n        scores = [fuzz.partial_ratio(sent, title) for title in ds_titles]\n        best_fit_title_index = np.argmax(scores)\n        if max(scores) > acceptance_score:\n            ds_candidates.add(ds_titles[np.argmax(scores)])\n    prediction_string = ' | '.join(ds_candidates)\n#     print (prediction_string)\n    test_df.loc[row.Index, 'PredictionString'] = prediction_string","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[['PredictionString']].to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cat submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Takeaways:\n\n**Immediate results**\n- generated set of sentences containing dataset titles.\n- established baseline of 93% accuracy with Naïve Bayes model.\n- NEW in v.3: achieved close to 99% acccuracy with Transformer model\n- NEW in v.5: atttempt to submit\n- NEW in v.6: test set processed consecutively to save RAM\n\n**Improvement opportunities**\n- improve sentence detection / splitting \n- test improvement with stemming\n- test potential improvement of sentence selection with models based on neural networks.\n\n**Further use:**\n- feed selected sentences to NER models for identification of the dataset titles\n\n**Files for reuse:**\n- `sentences_with_refs.pickle`, `sentences_empty.pickle` -- dicts with collected sentences\n- `sent_df.pickle` -- dataframe with sentences\n- `model folder` -- model weights","metadata":{}},{"cell_type":"markdown","source":"TBD:\n\nMultiprocess with SpaCy (thx to @lucabasa): https://stackoverflow.com/questions/44395656/applying-spacy-parser-to-pandas-dataframe-w-multiprocessing","metadata":{}}]}