{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Start writing code here...\nimport numpy as np \nimport pandas as pd\nimport nltk\n\npd.options.mode.chained_assignment = None \n\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import sent_tokenize\nfrom nltk.tag import pos_tag\nimport os, re\nimport json\nimport matplotlib.pyplot as plt\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.stem import WordNetLemmatizer\n  \nimport seaborn as sns\nsns.set_style('whitegrid')\n    \nfrom IPython.display import display, clear_output\n    \nfrom tqdm import tqdm\nimport string\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\n\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:00.748154Z","iopub.execute_input":"2021-06-13T14:51:00.748772Z","iopub.status.idle":"2021-06-13T14:51:02.608292Z","shell.execute_reply.started":"2021-06-13T14:51:00.748663Z","shell.execute_reply":"2021-06-13T14:51:02.606692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SPLIT = 0.9\nSENTENCES_TO_EXTRACT = 10\nDROP_FRAC = 0.8","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:02.610022Z","iopub.execute_input":"2021-06-13T14:51:02.610318Z","iopub.status.idle":"2021-06-13T14:51:02.614208Z","shell.execute_reply.started":"2021-06-13T14:51:02.610289Z","shell.execute_reply":"2021-06-13T14:51:02.613414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:02.615825Z","iopub.execute_input":"2021-06-13T14:51:02.616343Z","iopub.status.idle":"2021-06-13T14:51:02.951583Z","shell.execute_reply.started":"2021-06-13T14:51:02.616295Z","shell.execute_reply":"2021-06-13T14:51:02.950496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_text_ds = True\n\ntry:\n    CSV_PATH = \"../input/colerigde-processed-text/text.csv\"\n    pd.read_csv(CSV_PATH)\n    create_text_ds = False\nexcept:\n    print(\"No text dataset\")\n    CSV_PATH = 'text.csv'","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:02.953449Z","iopub.execute_input":"2021-06-13T14:51:02.953781Z","iopub.status.idle":"2021-06-13T14:51:09.305565Z","shell.execute_reply.started":"2021-06-13T14:51:02.95375Z","shell.execute_reply":"2021-06-13T14:51:09.304384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.306806Z","iopub.execute_input":"2021-06-13T14:51:09.307116Z","iopub.status.idle":"2021-06-13T14:51:09.338301Z","shell.execute_reply.started":"2021-06-13T14:51:09.307087Z","shell.execute_reply":"2021-06-13T14:51:09.337229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_df = pd.DataFrame(train_df['dataset_label'].value_counts())\ngrouped_df = grouped_df.reset_index()\ngrouped_df = grouped_df.sort_values(by=\"dataset_label\", ascending=False)\ngrouped_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.340059Z","iopub.execute_input":"2021-06-13T14:51:09.340817Z","iopub.status.idle":"2021-06-13T14:51:09.364966Z","shell.execute_reply.started":"2021-06-13T14:51:09.340764Z","shell.execute_reply":"2021-06-13T14:51:09.363875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_df = grouped_df.iloc[0:25]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.366461Z","iopub.execute_input":"2021-06-13T14:51:09.366763Z","iopub.status.idle":"2021-06-13T14:51:09.378556Z","shell.execute_reply.started":"2021-06-13T14:51:09.366734Z","shell.execute_reply":"2021-06-13T14:51:09.377311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,5))\n\nax.bar(grouped_df['index'].values, grouped_df['dataset_label'].values)\nax.set_xticklabels(labels=grouped_df['index'].values,rotation=90);\nax.set_ylabel(\"Number of occurances\")\nax.set_xlabel(\"Dataset name\")\n\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.381084Z","iopub.execute_input":"2021-06-13T14:51:09.381541Z","iopub.status.idle":"2021-06-13T14:51:09.898817Z","shell.execute_reply.started":"2021-06-13T14:51:09.381384Z","shell.execute_reply":"2021-06-13T14:51:09.897643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.900603Z","iopub.execute_input":"2021-06-13T14:51:09.901063Z","iopub.status.idle":"2021-06-13T14:51:09.932993Z","shell.execute_reply.started":"2021-06-13T14:51:09.901015Z","shell.execute_reply":"2021-06-13T14:51:09.931938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.934276Z","iopub.execute_input":"2021-06-13T14:51:09.934563Z","iopub.status.idle":"2021-06-13T14:51:09.952074Z","shell.execute_reply.started":"2021-06-13T14:51:09.934534Z","shell.execute_reply":"2021-06-13T14:51:09.950997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Nr. of unique labels: {}\".format(len(train_df['cleaned_label'].unique())))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.955276Z","iopub.execute_input":"2021-06-13T14:51:09.955604Z","iopub.status.idle":"2021-06-13T14:51:09.966533Z","shell.execute_reply.started":"2021-06-13T14:51:09.955573Z","shell.execute_reply":"2021-06-13T14:51:09.965757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for missing values\ntrain_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.96756Z","iopub.execute_input":"2021-06-13T14:51:09.968066Z","iopub.status.idle":"2021-06-13T14:51:09.98801Z","shell.execute_reply.started":"2021-06-13T14:51:09.968035Z","shell.execute_reply":"2021-06-13T14:51:09.987143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nr_train_csv_ids = train_df['Id'].count()\nnr_unique_train_csv_ids = train_df['Id'].nunique()\nnr_train_files = len([name for name in os.listdir(train_files_path) if os.path.isfile(os.path.join(train_files_path, name))])\n\n# Confirm that we have as many rows in the train.csv file as that we have files in the train.csv folder?\nprint(\"Data in train.csv file: {}\".format(nr_train_csv_ids))\nprint(\"\\n\")\nprint(\"Unique data in train.csv file: {}\".format(nr_unique_train_csv_ids))\nprint(\"Data in train folders: {}\".format(nr_train_files))\nprint(\"\\n\")\nprint(\"Avg. nr. of sources per publication: {}\".format(nr_train_csv_ids/nr_unique_train_csv_ids))\n\nassert nr_unique_train_csv_ids == nr_train_files","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:09.989038Z","iopub.execute_input":"2021-06-13T14:51:09.989457Z","iopub.status.idle":"2021-06-13T14:51:20.10819Z","shell.execute_reply.started":"2021-06-13T14:51:09.989426Z","shell.execute_reply":"2021-06-13T14:51:20.106765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords = []\nwith open(\"../input/stopwords/stopwords.txt\", \"r\") as fd:\n    stopwords = fd.read().splitlines()\n\nprint(stopwords)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:20.109573Z","iopub.execute_input":"2021-06-13T14:51:20.109936Z","iopub.status.idle":"2021-06-13T14:51:20.132545Z","shell.execute_reply.started":"2021-06-13T14:51:20.109884Z","shell.execute_reply":"2021-06-13T14:51:20.131526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list(train_df.values)\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:20.133874Z","iopub.execute_input":"2021-06-13T14:51:20.134216Z","iopub.status.idle":"2021-06-13T14:51:24.304753Z","shell.execute_reply.started":"2021-06-13T14:51:20.134182Z","shell.execute_reply":"2021-06-13T14:51:24.303998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1000, height=500, background_color='white', stopwords=STOPWORDS).generate(str(mostcommon))\nfig = plt.figure(figsize=(15,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in label')\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(15,5))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words')\nplt.ylabel('Frequency of Words')\nplt.xticks(rotation=60)\nplt.tight_layout(pad=0)\nplt.title('Freq of 25 Most Common Words in label')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:24.306094Z","iopub.execute_input":"2021-06-13T14:51:24.3067Z","iopub.status.idle":"2021-06-13T14:51:26.563152Z","shell.execute_reply.started":"2021-06-13T14:51:24.306652Z","shell.execute_reply":"2021-06-13T14:51:26.562376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:26.564272Z","iopub.execute_input":"2021-06-13T14:51:26.564733Z","iopub.status.idle":"2021-06-13T14:51:26.57818Z","shell.execute_reply.started":"2021-06-13T14:51:26.564679Z","shell.execute_reply":"2021-06-13T14:51:26.576737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retrieve document from random publication Id","metadata":{}},{"cell_type":"code","source":"pub_id = np.random.choice(train_df['Id'].unique())\npub_title = train_df[train_df[\"Id\"] == pub_id].pub_title\npub_df = pd.read_json(train_files_path+\"/\"+str(pub_id)+\".json\")\n\nprint(\"Random publication ID: {}\".format(pub_id))\nprint(\"Random publication name: {}\".format(pub_title))\nprint(\"\\n\")\n\npub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:51:30.110005Z","iopub.execute_input":"2021-06-13T14:51:30.110385Z","iopub.status.idle":"2021-06-13T14:51:30.166201Z","shell.execute_reply.started":"2021-06-13T14:51:30.110355Z","shell.execute_reply":"2021-06-13T14:51:30.165123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df[\"Id\"] == pub_id]","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.159797Z","iopub.execute_input":"2021-06-12T18:08:17.160104Z","iopub.status.idle":"2021-06-12T18:08:17.176298Z","shell.execute_reply.started":"2021-06-12T18:08:17.160073Z","shell.execute_reply":"2021-06-12T18:08:17.175263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub_dataset_titles = train_df[train_df[\"Id\"] == pub_id].dataset_title.values\n\nprint(\"Nr. of sections in this publication: {}\".format(pub_df[\"section_title\"].count()))\nprint(\"Datasets in this publication: {}\".format(pub_dataset_titles))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.177938Z","iopub.execute_input":"2021-06-12T18:08:17.178598Z","iopub.status.idle":"2021-06-12T18:08:17.197059Z","shell.execute_reply.started":"2021-06-12T18:08:17.178559Z","shell.execute_reply":"2021-06-12T18:08:17.196183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find sentence(s) containing dataset name\npub_df['lowercase_text'] = pub_df[\"text\"].str.lower()\n\npub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.200932Z","iopub.execute_input":"2021-06-12T18:08:17.2014Z","iopub.status.idle":"2021-06-12T18:08:17.22191Z","shell.execute_reply.started":"2021-06-12T18:08:17.201359Z","shell.execute_reply":"2021-06-12T18:08:17.220884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_datasets = np.array([])\n\nfor dataset in pub_dataset_titles:\n    clean_datasets = np.append(clean_datasets, dataset.lower())\n\nprint(clean_datasets)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.224131Z","iopub.execute_input":"2021-06-12T18:08:17.224626Z","iopub.status.idle":"2021-06-12T18:08:17.238183Z","shell.execute_reply.started":"2021-06-12T18:08:17.224588Z","shell.execute_reply":"2021-06-12T18:08:17.236695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_ds_sentence(text, dataset_title):\n    escaped_ds_title = re.escape(dataset_title)\n    return re.findall(r\"([^.]*?\"+re.escape(escaped_ds_title)+\"[^.]*\\.)\",text)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.241068Z","iopub.execute_input":"2021-06-12T18:08:17.241402Z","iopub.status.idle":"2021-06-12T18:08:17.254615Z","shell.execute_reply.started":"2021-06-12T18:08:17.241372Z","shell.execute_reply":"2021-06-12T18:08:17.253773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning the data","metadata":{}},{"cell_type":"code","source":"all_IDs = train_df['Id'].unique()\n\nprint(type(all_IDs))\nall_IDs_test = all_IDs[:5000]\nall_IDs_test[:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.255667Z","iopub.execute_input":"2021-06-12T18:08:17.256066Z","iopub.status.idle":"2021-06-12T18:08:17.276518Z","shell.execute_reply.started":"2021-06-12T18:08:17.256036Z","shell.execute_reply":"2021-06-12T18:08:17.275807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def slice_sentence(pub_id): \n    final_df = pd.DataFrame()\n    path_to_pub = os.path.join(train_files_path, (pub_id+'.json'))\n\n    sentences_col = np.array([])\n    labels_col = np.array([])\n    sections_col = np.array([])\n\n    with open(path_to_pub, 'r') as f:  \n        pub_df = pd.read_json(f)\n\n    for index,row in pub_df.iterrows():\n        sentences_in_section = re.findall(r\"[A-Z].*?[\\.!?]\\s?(?=[A-Z]|$)\",row[\"text\"])\n    \n        for sentence in sentences_in_section:\n            for label in train_df[train_df['Id'] == pub_id].dataset_label:\n                if label in sentence:\n                    sentences_col = np.append(sentences_col,sentence)\n                    labels_col = np.append(labels_col,label)\n                    sections_col = np.append(sections_col,row[\"section_title\"])\n    \n    final_df[\"Section\"] = sections_col\n    final_df[\"Text\"] = sentences_col\n    final_df[\"Label\"] = labels_col\n    final_df[\"Id\"] = pub_id\n    \n    return final_df","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.277612Z","iopub.execute_input":"2021-06-12T18:08:17.278026Z","iopub.status.idle":"2021-06-12T18:08:17.287042Z","shell.execute_reply.started":"2021-06-12T18:08:17.277992Z","shell.execute_reply":"2021-06-12T18:08:17.28609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def slice_section(pub_id): \n    final_df = pd.DataFrame()\n    path_to_pub = os.path.join(train_files_path, (pub_id+'.json'))\n\n    text_col = np.array([])\n    labels_col = np.array([])\n    sections_col = np.array([])\n\n    with open(path_to_pub, 'r') as f:  \n        pub_df = pd.read_json(f)\n\n    for index,row in pub_df.iterrows():\n        for label in train_df[train_df['Id'] == pub_id].dataset_label:\n            if label in row['text']:\n                labels_col = np.append(labels_col,label)\n                sections_col = np.append(sections_col,row[\"section_title\"])\n                text_col = np.append(text_col,row[\"text\"])\n            \n    \n    final_df[\"Section\"] = sections_col\n    final_df[\"Text\"] = text_col\n    final_df[\"Label\"] = labels_col\n    final_df[\"Id\"] = pub_id\n    \n    return final_df","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.288716Z","iopub.execute_input":"2021-06-12T18:08:17.289073Z","iopub.status.idle":"2021-06-12T18:08:17.312111Z","shell.execute_reply.started":"2021-06-12T18:08:17.289026Z","shell.execute_reply":"2021-06-12T18:08:17.310705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slice_section(all_IDs[13])","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.313758Z","iopub.execute_input":"2021-06-12T18:08:17.31425Z","iopub.status.idle":"2021-06-12T18:08:17.443484Z","shell.execute_reply.started":"2021-06-12T18:08:17.314217Z","shell.execute_reply":"2021-06-12T18:08:17.441714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(slice_section(all_IDs[13])['Text'].values[0])\nprint(slice_section(all_IDs[13])['Label'].values[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.44472Z","iopub.execute_input":"2021-06-12T18:08:17.445046Z","iopub.status.idle":"2021-06-12T18:08:17.627087Z","shell.execute_reply.started":"2021-06-12T18:08:17.445017Z","shell.execute_reply":"2021-06-12T18:08:17.626037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slice_section(\"83c59eeb-d015-4c4e-8d36-cb2cc6dadbe2\")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.629237Z","iopub.execute_input":"2021-06-12T18:08:17.629507Z","iopub.status.idle":"2021-06-12T18:08:17.772992Z","shell.execute_reply.started":"2021-06-12T18:08:17.629481Z","shell.execute_reply":"2021-06-12T18:08:17.771798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_contents = np.array([])","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.774456Z","iopub.execute_input":"2021-06-12T18:08:17.774738Z","iopub.status.idle":"2021-06-12T18:08:17.778434Z","shell.execute_reply.started":"2021-06-12T18:08:17.774712Z","shell.execute_reply":"2021-06-12T18:08:17.777694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if create_text_ds:\n    # Remove file if already exists\n    if os.path.exists(CSV_PATH):\n        os.remove(CSV_PATH)\n\n    print(\"Starting text slicing\")\n\n    for i,filename in enumerate(tqdm(all_IDs, desc='Slicing text')):  \n        file_df = slice_section(filename)\n\n        if i==0: file_df.to_csv(CSV_PATH, mode='a', index=False, header=True)\n        else: file_df.to_csv(CSV_PATH, mode='a', index=False, header=False)\n\n        # comment when using all data\n#         if i==500: break","metadata":{"execution":{"iopub.status.busy":"2021-06-12T18:08:17.779746Z","iopub.execute_input":"2021-06-12T18:08:17.780255Z","iopub.status.idle":"2021-06-12T18:08:17.794825Z","shell.execute_reply.started":"2021-06-12T18:08:17.780214Z","shell.execute_reply":"2021-06-12T18:08:17.793751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_df = pd.read_csv(CSV_PATH)\n\nprint(clean_df.shape)\nclean_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:12.542085Z","iopub.execute_input":"2021-06-12T20:56:12.542569Z","iopub.status.idle":"2021-06-12T20:56:15.44457Z","shell.execute_reply.started":"2021-06-12T20:56:12.542538Z","shell.execute_reply":"2021-06-12T20:56:15.443381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(clean_df['Text'].values[0])\nprint(clean_df['Label'].values[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:15.44642Z","iopub.execute_input":"2021-06-12T20:56:15.446834Z","iopub.status.idle":"2021-06-12T20:56:15.453651Z","shell.execute_reply.started":"2021-06-12T20:56:15.44679Z","shell.execute_reply":"2021-06-12T20:56:15.452467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:15.455967Z","iopub.execute_input":"2021-06-12T20:56:15.456384Z","iopub.status.idle":"2021-06-12T20:56:15.46762Z","shell.execute_reply.started":"2021-06-12T20:56:15.456341Z","shell.execute_reply":"2021-06-12T20:56:15.466623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def find_word_in_text(text, word):\n#     matches = []\n#     if text.find(word) != -1: # When substring is found\n#         pattern = re.compile(r\"\\b(\\w*\"+re.escape(word)+r\"\\w*)\\b\")\n        \n#         for match in pattern.finditer(text):\n#             matches.append((match.start(),match.end()))\n            \n                           \n#     return matches","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:15.469596Z","iopub.execute_input":"2021-06-12T20:56:15.470277Z","iopub.status.idle":"2021-06-12T20:56:15.4803Z","shell.execute_reply.started":"2021-06-12T20:56:15.470232Z","shell.execute_reply":"2021-06-12T20:56:15.479298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word = \"national education longitudinal study\" \ntext = \"this study used data from. the national education longitudinal studys nels 88 to examine the effects of dual enrollment. programs for high school students on college degree attainment the study also reported whether the impacts of dual enrollment programs were different for first generation college students versus students whose parents had attended at. least national education longitudinal study some college in addition a supplemental analysis reports on the impact of different amounts of dual enrollment course taking and college degree attainment dual enrollment programs offer college level learning experiences for high school students the programs offer college courses and or the opportunity to earn college credits for students while still in high school the intervention group in the study was comprised of nels participants who attended a postsecondary school and who participated in a dual enrollment. program while in high school n 880 the study. author used propensity score mat\"","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:15.48203Z","iopub.execute_input":"2021-06-12T20:56:15.482444Z","iopub.status.idle":"2021-06-12T20:56:15.492463Z","shell.execute_reply.started":"2021-06-12T20:56:15.482404Z","shell.execute_reply":"2021-06-12T20:56:15.49131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cut_sentences(text, word, nr_of_sentences):\n    full_matches = []\n    full_texts = []\n    \n    tokens = nltk.sent_tokenize(text)\n\n    for i,t in enumerate(tokens):\n        matches = []\n        nr = nr_of_sentences\n        start_sentence = i\n        end_sentence = i\n        if t.find(word) != -1: # When substring is found               \n            start = True\n            end = False\n\n            while nr > -1:\n                if start:\n                    if start_sentence != 0:\n                        start_sentence = start_sentence - 1\n\n                    start = False\n                    end = True\n                    \n                if end:\n                    if end_sentence != len(tokens):\n                        end_sentence = end_sentence + 1\n                        \n                    start = True\n                    end = False\n            \n                nr = nr - 1\n                \n            text = tokens[start_sentence:end_sentence]\n\n            text = ' '.join(text)\n        \n            pattern = re.compile(r\"\\b(\\S*\"+re.escape(word)+r\"\\S*)\")\n    \n            for match in pattern.finditer(text):\n                matches.append((match.start(),match.end(),\"DATASET\"))\n                \n            full_matches.append(matches)\n            full_texts.append(text)\n                        \n    return {\"texts\": full_texts, \"matches\":full_matches}","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:15.493756Z","iopub.execute_input":"2021-06-12T20:56:15.494388Z","iopub.status.idle":"2021-06-12T20:56:15.5169Z","shell.execute_reply.started":"2021-06-12T20:56:15.494344Z","shell.execute_reply":"2021-06-12T20:56:15.515558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = cut_sentences(text,word,5)\n\nfor text in example['texts']:\n    print(text)\n    \nfor match in example['matches']:\n    print(match)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:16.0074Z","iopub.execute_input":"2021-06-12T20:56:16.007964Z","iopub.status.idle":"2021-06-12T20:56:16.014384Z","shell.execute_reply.started":"2021-06-12T20:56:16.007929Z","shell.execute_reply":"2021-06-12T20:56:16.01359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\n\ntrain_df = pd.DataFrame(columns=clean_df.columns)\n\nfor idxs, row in tqdm(clean_df.iterrows()):\n    temp_df = pd.DataFrame(columns=clean_df.columns)\n    \n    cuts = cut_sentences(row['Text'], row['Label'], SENTENCES_TO_EXTRACT)\n    \n    temp_df['Text'] = cuts['texts']\n    temp_df['dataset_index'] = cuts['matches']\n    \n    temp_df['Id'] = row['Id']\n    temp_df['Section'] = row['Section']\n    temp_df['Label'] = row['Label']\n    \n    train_df = train_df.append(temp_df)\n    \n#     if i == 1:\n#         break;\n        \n#     i = i+1","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:56:16.610493Z","iopub.execute_input":"2021-06-12T20:56:16.610959Z","iopub.status.idle":"2021-06-12T21:03:28.573528Z","shell.execute_reply.started":"2021-06-12T20:56:16.610928Z","shell.execute_reply":"2021-06-12T21:03:28.572805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:03:28.57512Z","iopub.execute_input":"2021-06-12T21:03:28.575433Z","iopub.status.idle":"2021-06-12T21:03:28.591979Z","shell.execute_reply.started":"2021-06-12T21:03:28.575404Z","shell.execute_reply":"2021-06-12T21:03:28.590988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df.explode('dataset_index')\n# train_df = train_df.drop_duplicates()\ntrain_df = train_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:02.340755Z","iopub.execute_input":"2021-06-12T21:04:02.341116Z","iopub.status.idle":"2021-06-12T21:04:02.397271Z","shell.execute_reply.started":"2021-06-12T21:04:02.341084Z","shell.execute_reply":"2021-06-12T21:04:02.396465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.reset_index(drop=True)\nprint(train_df.shape)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:08.330917Z","iopub.execute_input":"2021-06-12T21:04:08.331494Z","iopub.status.idle":"2021-06-12T21:04:08.357215Z","shell.execute_reply.started":"2021-06-12T21:04:08.331444Z","shell.execute_reply":"2021-06-12T21:04:08.356034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_group = pd.DataFrame(train_df['Label'].value_counts())\ndf_group = df_group.reset_index()\ndf_group = df_group.sort_values(by=\"Label\", ascending=False)\n\ndf_group.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:14.6476Z","iopub.execute_input":"2021-06-12T21:04:14.647995Z","iopub.status.idle":"2021-06-12T21:04:14.673434Z","shell.execute_reply.started":"2021-06-12T21:04:14.647926Z","shell.execute_reply":"2021-06-12T21:04:14.672449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Dataset with highest number of labels: {}\".format(df_group.values[0][0]))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:16.859655Z","iopub.execute_input":"2021-06-12T21:04:16.860007Z","iopub.status.idle":"2021-06-12T21:04:16.865885Z","shell.execute_reply.started":"2021-06-12T21:04:16.859978Z","shell.execute_reply":"2021-06-12T21:04:16.864964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Removing {} labels\".format(df_group.values[0][1] * DROP_FRAC))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:18.672365Z","iopub.execute_input":"2021-06-12T21:04:18.673056Z","iopub.status.idle":"2021-06-12T21:04:18.67943Z","shell.execute_reply.started":"2021-06-12T21:04:18.673005Z","shell.execute_reply":"2021-06-12T21:04:18.678373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove some of the highest occuring values to prevent memoryerror during training\nindices = train_df.index[train_df['Label'] == df_group.values[0][0]]\n\ntrain_df.drop(indices[:int(len(indices) * DROP_FRAC)], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:20.896074Z","iopub.execute_input":"2021-06-12T21:04:20.896441Z","iopub.status.idle":"2021-06-12T21:04:20.917188Z","shell.execute_reply.started":"2021-06-12T21:04:20.896411Z","shell.execute_reply":"2021-06-12T21:04:20.916211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove sentences that are too large for the model to train\nmean = np.array([])\ncount = 0\nmean_limit = 100\n\nfor idx, rows in train_df.iterrows():\n    mean = np.append(mean, len(rows['Text'].split()))\n    \n    if len(rows['Text'].split()) > (mean.mean() * mean_limit):\n        print(\"Removing row with length {} using limit {}\".format(len(rows['Text'].split()), (mean.mean() * mean_limit)))\n        train_df.drop(idx, inplace=True)\n        count = count + 1\n\nprint(\"Removed {} rows\".format(count))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:24.824563Z","iopub.execute_input":"2021-06-12T21:04:24.824934Z","iopub.status.idle":"2021-06-12T21:04:33.830334Z","shell.execute_reply.started":"2021-06-12T21:04:24.824898Z","shell.execute_reply":"2021-06-12T21:04:33.829376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(r\"\\b(\\w*\"+re.escape(word)+r\"\\w*)\\b\")\n\nfor match in pattern.finditer(text):\n    print(match.start())\n    print(match.end())","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:36.869655Z","iopub.execute_input":"2021-06-12T21:04:36.870004Z","iopub.status.idle":"2021-06-12T21:04:36.878695Z","shell.execute_reply.started":"2021-06-12T21:04:36.869974Z","shell.execute_reply":"2021-06-12T21:04:36.877724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_indexes = 0\n\nfor idxs, rows in train_df.iterrows():\n    for i in rows['dataset_index']:\n        if i == (0, 0, \"DATASET\"):\n            missing_indexes = missing_indexes + 1\n            \nprint(\"Found \"+str(missing_indexes)+\" empty indexes\")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:38.177394Z","iopub.execute_input":"2021-06-12T21:04:38.177729Z","iopub.status.idle":"2021-06-12T21:04:42.317156Z","shell.execute_reply.started":"2021-06-12T21:04:38.177702Z","shell.execute_reply":"2021-06-12T21:04:42.316029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df.groupby('Label')['dataset_index'].count().to_frame()\ndf = df[df['dataset_index'] == 1]\n\nprint(\"Found {} single dataset labels\".format(len(df['dataset_index'].values)))\n\nsingle_labels = df.index.values\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:44.294919Z","iopub.execute_input":"2021-06-12T21:04:44.295293Z","iopub.status.idle":"2021-06-12T21:04:44.321858Z","shell.execute_reply.started":"2021-06-12T21:04:44.295263Z","shell.execute_reply":"2021-06-12T21:04:44.321131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def duplicate_single_labels(row):\n    if row['Label'] in single_labels:\n        row2 = row.copy()\n        # make edits to row2\n        return pd.concat([row, row2], axis=1)\n    return row","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:46.25576Z","iopub.execute_input":"2021-06-12T21:04:46.256498Z","iopub.status.idle":"2021-06-12T21:04:46.261128Z","shell.execute_reply.started":"2021-06-12T21:04:46.256462Z","shell.execute_reply":"2021-06-12T21:04:46.260294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([duplicate_single_labels(row) for _, row in train_df.iterrows()], ignore_index=True, axis=1).T\nprint(train_df.shape)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:04:47.733284Z","iopub.execute_input":"2021-06-12T21:04:47.733782Z","iopub.status.idle":"2021-06-12T21:05:12.663862Z","shell.execute_reply.started":"2021-06-12T21:04:47.733752Z","shell.execute_reply":"2021-06-12T21:05:12.663186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['dataset_index'].values[0][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:12.665035Z","iopub.execute_input":"2021-06-12T21:05:12.665322Z","iopub.status.idle":"2021-06-12T21:05:12.671381Z","shell.execute_reply.started":"2021-06-12T21:05:12.665295Z","shell.execute_reply":"2021-06-12T21:05:12.670224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This code checks whether the dataset label is actually present in the text given the generated dataset indices\n# If this is not the case, an assertion error will be raised and the text will not further be processed\nfor idx, row in tqdm(train_df.iterrows()):\n    for i in row['dataset_index']:\n        assert row['Text'].count(row['Label'],\n                                 i[0],\n                                 i[1]) != 0\n        assert len(row['dataset_index']) != 0","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:20.976052Z","iopub.execute_input":"2021-06-12T21:05:20.976433Z","iopub.status.idle":"2021-06-12T21:05:26.777923Z","shell.execute_reply.started":"2021-06-12T21:05:20.976401Z","shell.execute_reply":"2021-06-12T21:05:26.777268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=42) # Shuffle the dataset\ntrain_df = train_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:28.629694Z","iopub.execute_input":"2021-06-12T21:05:28.630275Z","iopub.status.idle":"2021-06-12T21:05:28.651776Z","shell.execute_reply.started":"2021-06-12T21:05:28.630226Z","shell.execute_reply":"2021-06-12T21:05:28.651008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:30.481391Z","iopub.execute_input":"2021-06-12T21:05:30.481896Z","iopub.status.idle":"2021-06-12T21:05:30.487959Z","shell.execute_reply.started":"2021-06-12T21:05:30.481854Z","shell.execute_reply":"2021-06-12T21:05:30.486777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:32.965231Z","iopub.execute_input":"2021-06-12T21:05:32.965696Z","iopub.status.idle":"2021-06-12T21:05:32.981491Z","shell.execute_reply.started":"2021-06-12T21:05:32.965667Z","shell.execute_reply":"2021-06-12T21:05:32.980505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df['Label'].to_frame()\nX = train_df","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:35.071306Z","iopub.execute_input":"2021-06-12T21:05:35.071787Z","iopub.status.idle":"2021-06-12T21:05:35.076832Z","shell.execute_reply.started":"2021-06-12T21:05:35.071756Z","shell.execute_reply":"2021-06-12T21:05:35.07603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:35.722341Z","iopub.execute_input":"2021-06-12T21:05:35.722903Z","iopub.status.idle":"2021-06-12T21:05:35.740072Z","shell.execute_reply.started":"2021-06-12T21:05:35.72287Z","shell.execute_reply":"2021-06-12T21:05:35.738999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:37.588089Z","iopub.execute_input":"2021-06-12T21:05:37.588451Z","iopub.status.idle":"2021-06-12T21:05:37.597601Z","shell.execute_reply.started":"2021-06-12T21:05:37.588422Z","shell.execute_reply":"2021-06-12T21:05:37.596618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, validation = train_test_split(X, train_size=TRAIN_SPLIT, random_state=42, stratify=y)\nprint(\"Train and validation set generated\")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:39.071667Z","iopub.execute_input":"2021-06-12T21:05:39.072053Z","iopub.status.idle":"2021-06-12T21:05:39.523673Z","shell.execute_reply.started":"2021-06-12T21:05:39.072021Z","shell.execute_reply":"2021-06-12T21:05:39.522543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reset_index(drop=True)\n\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:54.355182Z","iopub.execute_input":"2021-06-12T21:05:54.355577Z","iopub.status.idle":"2021-06-12T21:05:54.376488Z","shell.execute_reply.started":"2021-06-12T21:05:54.355542Z","shell.execute_reply":"2021-06-12T21:05:54.375487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation = validation.reset_index(drop=True)\n\nprint(validation.shape)\nvalidation.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:05:58.92341Z","iopub.execute_input":"2021-06-12T21:05:58.923798Z","iopub.status.idle":"2021-06-12T21:05:58.943956Z","shell.execute_reply.started":"2021-06-12T21:05:58.923753Z","shell.execute_reply":"2021-06-12T21:05:58.94296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train = pd.DataFrame(train['Label'].value_counts(normalize=True))\ngrouped_train = grouped_train.reset_index()\ngrouped_train = grouped_train.sort_values(by=\"Label\", ascending=False)\n\ngrouped_train = grouped_train.iloc[0:25]\n\ngrouped_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:06:01.010042Z","iopub.execute_input":"2021-06-12T21:06:01.010448Z","iopub.status.idle":"2021-06-12T21:06:01.03574Z","shell.execute_reply.started":"2021-06-12T21:06:01.010413Z","shell.execute_reply":"2021-06-12T21:06:01.03474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_val = pd.DataFrame(validation['Label'].value_counts(normalize=True))\ngrouped_val = grouped_val.reset_index()\ngrouped_val = grouped_val.sort_values(by=\"Label\", ascending=False)\n\ngrouped_val = grouped_val.iloc[0:25]\n\ngrouped_val.head(25)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:06:03.489898Z","iopub.execute_input":"2021-06-12T21:06:03.490307Z","iopub.status.idle":"2021-06-12T21:06:03.510853Z","shell.execute_reply.started":"2021-06-12T21:06:03.49027Z","shell.execute_reply":"2021-06-12T21:06:03.509621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numbers of pairs of bars you want\nN = 25\n\n# Position of bars on x-axis\nind = np.arange(N)\n\n# Figure size\nplt.figure(figsize=(10,5))\n\n# Width of a bar \nwidth = 0.3       \n\n# Plotting\nplt.bar(ind, grouped_train['Label'].values, width, label='Train distribution')\nplt.bar(ind + width, grouped_val['Label'].values, width, label='Validation distribution')\n\nplt.xlabel('Dataset name index')\nplt.ylabel('Number of occurances (normalized)')\n\nplt.legend(loc='best')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:06:05.647864Z","iopub.execute_input":"2021-06-12T21:06:05.648237Z","iopub.status.idle":"2021-06-12T21:06:06.006688Z","shell.execute_reply.started":"2021-06-12T21:06:05.648193Z","shell.execute_reply":"2021-06-12T21:06:06.005718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv(\"./train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:06:08.434907Z","iopub.execute_input":"2021-06-12T21:06:08.435299Z","iopub.status.idle":"2021-06-12T21:06:10.367327Z","shell.execute_reply.started":"2021-06-12T21:06:08.43526Z","shell.execute_reply":"2021-06-12T21:06:10.366454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.to_csv(\"./validation.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T21:06:10.368433Z","iopub.execute_input":"2021-06-12T21:06:10.368701Z","iopub.status.idle":"2021-06-12T21:06:10.474539Z","shell.execute_reply.started":"2021-06-12T21:06:10.368674Z","shell.execute_reply":"2021-06-12T21:06:10.473657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}