{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Sentences dataframe for coleridge NER competition  \nThis notebook creates a sentences df starting from the section extended dataset created [here](https://www.kaggle.com/davidemariani/coleridge-ner-extended-df)  \nPlease upvote if you find this useful!","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport seaborn as sns\nimport re\nimport nltk\n\nfrom nltk.tokenize import sent_tokenize, word_tokenize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MAIN SETTINGS\nmax_sentence_length = 60 #max n. of words for each slice of text\noverlap = 20 #number of overlapping words in case a sentence is broken in more sentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_pickle(\"../input/coleridge-ner-full-info-df/coleridge_train_extended.pkl\")\n\ntrain_df = train_df[['Id', 'pub_title', 'dataset_title', 'dataset_label',\n       'cleaned_label', 'section_title', 'section_number', 'text', 'cleaned_text',\n       'label_match']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing rows with null text\nprint(train_df.shape)\ntrain_df = train_df.dropna(subset = ['text'])\nprint(train_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_by_id = train_df.groupby(['Id', 'text']).dataset_label.unique()\nlabels_by_id_cleaned = train_df.groupby(['Id', 'text']).cleaned_label.unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_by_id_list = []\nlabels_by_id_list_cleaned = []\n\nfor i in range(train_df.shape[0]):\n    labels_by_id_list.append(labels_by_id[train_df.iloc[i].Id, train_df.iloc[i].text])\n    labels_by_id_list_cleaned.append(labels_by_id_cleaned[train_df.iloc[i].Id, train_df.iloc[i].text])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#column with the list of dataset included in the paper\ntrain_df['dataset_label_in_id'] = labels_by_id_list\ntrain_df['dataset_label_in_id_cleaned'] = labels_by_id_list_cleaned","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing duplicate texts\ntrain_df = train_df.drop_duplicates('text')\ntrain_df = train_df.reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#column with the list of dataset included in the specific text\ntrain_df['dataset_label_in_text'] = train_df.apply(lambda x:[j for j in x.dataset_label_in_id if j in x.text], axis=1)\ntrain_df['dataset_label_in_text_cleaned'] = train_df.apply(lambda x:[j for j in x.dataset_label_in_id_cleaned if j in x.cleaned_text], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"There are {} texts with entities over {} - about {}%\".format(train_df.label_match.sum(), train_df.shape[0], round(train_df.label_match.sum()*100/ train_df.shape[0])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def break_sentence(sentence, max_sentence_length, overlap):\n    \n    words = sentence.split()\n    \n    sentence_length = len(words)\n    \n    if sentence_length <= max_sentence_length:\n        return [sentence]\n    \n    else:\n        broken_sentences = []\n        \n        for p in range(0, sentence_length, max_sentence_length - overlap):\n            broken_sentences.append(\" \".join(words[p:p + max_sentence_length]))\n            \n        return broken_sentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating sentences dataframe\ns_dict = {}\n\ns_dict['Id'] = []\ns_dict['sentence_id'] = []\ns_dict['pub_title'] = []\ns_dict['text'] = []\ns_dict['cleaned_text'] = []\ns_dict['section_title'] = []\ns_dict['section_number'] = []\ns_dict['dataset_label_in_id'] = []\ns_dict['dataset_label_in_id_cleaned'] = []\ns_dict['dataset_label_in_text'] = []\ns_dict['dataset_label_in_text_cleaned'] = []\ns_dict['has_reference'] = []\ns_dict['has_reference_cleaned'] = []\ns_dict['n_words'] = []\n\n\ncurrent_id = ''\n\nfor t in range(train_df.shape[0]):\n    slice_df = train_df.iloc[t]\n    \n    pub_id = slice_df.Id\n    \n    if current_id != pub_id:\n        count = 1\n        current_id = pub_id\n    \n    for sup_s in sent_tokenize(slice_df.text):\n        \n        for s in break_sentence(sup_s, max_sentence_length, overlap):\n        \n            s_dict['Id'].append(pub_id)\n            s_dict['pub_title'].append(slice_df.pub_title)\n            s_dict['text'].append(s)\n\n            if count < 10:\n                strcount = \"000\" + str(count)\n\n            elif 10 <= count < 100:\n                strcount = \"00\" + str(count)\n\n            elif 100 <= count < 1000:\n                strcount = \"0\" + str(count)\n\n            else:\n                strcount = str(count)\n\n            s_dict['sentence_id'].append(slice_df.Id + '_' + strcount)\n\n            c_text = clean_text(s)\n            s_dict['cleaned_text'].append(c_text)\n\n            s_dict['section_title'].append(slice_df.section_title)\n            s_dict['section_number'].append(slice_df.section_number)\n            s_dict['dataset_label_in_id'].append(slice_df.dataset_label_in_id)\n            s_dict['dataset_label_in_id_cleaned'].append(slice_df.dataset_label_in_id_cleaned)\n\n            ds_matches = []\n            if len(slice_df.dataset_label_in_text) > 0:\n                for ds in slice_df.dataset_label_in_text:\n                    if ds in s:\n                        ds_matches.append(ds)\n\n            ds_matches_cleaned = []\n            if len(slice_df.dataset_label_in_text_cleaned) > 0:\n                for ds_c in slice_df.dataset_label_in_text_cleaned:\n                    if ds_c in c_text:\n                        ds_matches_cleaned.append(ds_c)\n\n            s_dict['dataset_label_in_text'].append(ds_matches)\n            s_dict['dataset_label_in_text_cleaned'].append(ds_matches_cleaned)\n\n            s_dict['has_reference'].append(len(ds_matches) > 0)\n            s_dict['has_reference_cleaned'].append(len(ds_matches_cleaned) > 0)\n\n            s_dict['n_words'].append(len(s.split()))\n\n            count+=1\n\n        \n        \n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df = pd.DataFrame(s_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_df.to_pickle(\"coleridge_sentence_df.pkl\") #.to_csv(\"coleridge_sentence_df.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}