{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Data Analysis \nAnalysis of the given dataset, including bag of word for sentences in which dataset labels are mentioned and N-Grams. ","metadata":{}},{"cell_type":"markdown","source":"# Loading the data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nimport json\nimport re\nimport os\n\nINPUT_PATH = '/kaggle/input/coleridgeinitiative-show-us-the-data'\nWORKING_PATH = '/kaggle/working'","metadata":{"execution":{"iopub.status.busy":"2021-06-12T11:53:11.518968Z","iopub.execute_input":"2021-06-12T11:53:11.519516Z","iopub.status.idle":"2021-06-12T11:53:11.524817Z","shell.execute_reply.started":"2021-06-12T11:53:11.519431Z","shell.execute_reply":"2021-06-12T11:53:11.523732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code of this cell base on https://www.kaggle.com/hsbota/simple-baseline-finding-labels-in-text-hbot\ndef clean_text(txt):\n return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef read_doc(doc_id, split:str = 'train'):\n path = os.path.join(INPUT_PATH, split, f'{doc_id}.json')\n return json.loads(open(path).read())\n\n \ndef row_to_doc_text_uncleaned(row: pd.Series, split:str = 'train') -> str:\n \n doc_id = row['Id']\n doc_title = row['pub_title'] if 'pub_title' in row else ''\n \n doc_content = read_doc(doc_id, split)\n doc_text = ' '.join([section['section_title'] + ':. ' + \n section['text'] for section in doc_content])\n \n return doc_text","metadata":{"execution":{"iopub.status.busy":"2021-06-12T11:53:11.530024Z","iopub.execute_input":"2021-06-12T11:53:11.530458Z","iopub.status.idle":"2021-06-12T11:53:11.539793Z","shell.execute_reply.started":"2021-06-12T11:53:11.530369Z","shell.execute_reply":"2021-06-12T11:53:11.538620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(INPUT_PATH, 'train.csv'))\ntrain_df['pub_text'] = train_df.apply(row_to_doc_text_uncleaned, split='train', axis='columns')\nprint(train_df.nunique())\nprint(\"Sample of the dataset, including the raw text\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T11:53:11.541010Z","iopub.execute_input":"2021-06-12T11:53:11.541409Z","iopub.status.idle":"2021-06-12T11:54:20.416291Z","shell.execute_reply.started":"2021-06-12T11:53:11.541380Z","shell.execute_reply":"2021-06-12T11:54:20.415242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Amount of unique Datset labels: \", len(train_df['dataset_label'].value_counts().index))\nprint(\"Top 30 Mentioned Datasets: \")\ndataset_label_counts = train_df['dataset_label'].value_counts(dropna=False)\ndf_dataset_counts = pd.DataFrame(dataset_label_counts)\ndf_dataset_counts = df_dataset_counts.reset_index()\ndf_dataset_counts.columns = ['dataset_label','dataset_frequency'] \nprint(df_dataset_counts.head(30))\n\nprint(\"Interessting observation: SARS-CoV-2 genome sequence is treated as a Dataset \")","metadata":{"execution":{"iopub.status.busy":"2021-06-12T11:54:20.418858Z","iopub.execute_input":"2021-06-12T11:54:20.419284Z","iopub.status.idle":"2021-06-12T11:54:20.444409Z","shell.execute_reply.started":"2021-06-12T11:54:20.419239Z","shell.execute_reply":"2021-06-12T11:54:20.443289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract sentences that contain the labeled dataset","metadata":{}},{"cell_type":"code","source":"# text is not cleaned, i.e. punctation, capitals and further are preserved\n\nmentions_sentences = []\nmentions_ds = []\nmentions_id = []\nmentions_index = []\nmentions_three_gram = []\nmentions_two_gram = []\nmentions_one_gram = []\nmentions_nthree_gram = []\nmentions_ntwo_gram = []\nmentions_none_gram = []\nnot_found_label = []\nnot_found_id = []\n\n\nfor i in range(len(train_df.index)):\n # split after sentences end (\".\", \"!\" or \"?\" is followed by whitespace and captial letter)\n text = re.split(\"(?<=[\\.\\!\\?])\\s*(?=[A-Z])\", train_df['pub_text'][i])\n for sentence in text: \n # if the according label is substring of the sentence\n if train_df['dataset_label'][i] in sentence:\n # add sentence,label and Id to database\n mentions_sentences.append(sentence)\n mentions_ds.append(train_df['dataset_label'][i])\n mentions_id.append(train_df['Id'][i])\n \n # create seperate strings for each word\n word_list = re.sub('[\\,\\;\\:\\.\\!\\?]', '', sentence).split()\n dataset_strings = train_df['dataset_label'][i].split()\n indicies = []\n #find indicies of mentioned database in sentence\n for dataset_string in dataset_strings:\n # if words of dataset are found as whole strings\n if dataset_string not in word_list:\n #see if they can be found as sub strings\n for idx,string in enumerate(word_list):\n if dataset_string in string:\n indicies.append(idx)\n else:\n indicies.append(word_list.index(dataset_string))\n # save index of first dataset word \n mentions_index.append(indicies[0])\n \n # create 3-gram\n if indicies[0]>2:\n three_gram= word_list[indicies[0]-3] + \" \" +word_list[indicies[0]-2] + \" \" + word_list[indicies[0]-1]\n mentions_three_gram.append(three_gram)\n else: \n mentions_three_gram.append(None)\n \n # create 2-gram\n if indicies[0]>1:\n two_gram= word_list[indicies[0]-2] + \" \" + word_list[indicies[0]-1]\n mentions_two_gram.append(two_gram)\n else: \n mentions_two_gram.append(None)\n \n # create 1-gram\n if indicies[0]>0:\n one_gram= word_list[indicies[0]-1]\n mentions_one_gram.append(one_gram)\n else: \n mentions_one_gram.append(None)\n \n # create -3-gram\n if indicies[-1]