{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-03T08:42:05.115104Z","iopub.execute_input":"2021-06-03T08:42:05.115496Z","iopub.status.idle":"2021-06-03T08:42:19.746056Z","shell.execute_reply.started":"2021-06-03T08:42:05.11546Z","shell.execute_reply":"2021-06-03T08:42:19.744621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\nimport re\nfrom pickle import dump, load\nimport json\nimport time\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:42:19.747574Z","iopub.execute_input":"2021-06-03T08:42:19.747976Z","iopub.status.idle":"2021-06-03T08:42:21.418702Z","shell.execute_reply.started":"2021-06-03T08:42:19.747943Z","shell.execute_reply":"2021-06-03T08:42:21.417851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers.embeddings import Embedding\nfrom keras.preprocessing.text import text_to_word_sequence, Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nfrom numpy import asarray\nfrom numpy import zeros","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:42:21.420661Z","iopub.execute_input":"2021-06-03T08:42:21.420992Z","iopub.status.idle":"2021-06-03T08:42:27.574216Z","shell.execute_reply.started":"2021-06-03T08:42:21.420962Z","shell.execute_reply":"2021-06-03T08:42:27.573071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merging ``train.csv`` with the rights articles","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-03T08:42:27.575754Z","iopub.execute_input":"2021-06-03T08:42:27.576102Z","iopub.status.idle":"2021-06-03T08:42:27.723537Z","shell.execute_reply.started":"2021-06-03T08:42:27.57607Z","shell.execute_reply":"2021-06-03T08:42:27.722537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following function will be used to perform compile all paragraphs from a given article (using the ``Id`` column from the ``train.csv`` file) and then concatenate it with our ``train`` dataframe (``train.csv``). ","metadata":{}},{"cell_type":"code","source":"def merge(filename, test=False):\n    if test:\n        df = pd.read_json(f'../input/coleridgeinitiative-show-us-the-data/test/{filename}.json')\n    else:\n        df = pd.read_json(f'../input/coleridgeinitiative-show-us-the-data/train/{filename}.json')\n    text = \" \".join(list(df['text']))\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:42:27.724889Z","iopub.execute_input":"2021-06-03T08:42:27.725205Z","iopub.status.idle":"2021-06-03T08:42:27.730768Z","shell.execute_reply.started":"2021-06-03T08:42:27.725173Z","shell.execute_reply":"2021-06-03T08:42:27.729466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we apply this function to our ``train.csv`` file to concatenate all our training data.","metadata":{}},{"cell_type":"code","source":"start = time.time()\ntrain['text'] = train['Id'].apply(merge)\nend = time.time()\nprint(f'This cell executes in {end - start:.2f} seconds.') # usually 1.5mn - but sometimes longer","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:42:27.732461Z","iopub.execute_input":"2021-06-03T08:42:27.732906Z","iopub.status.idle":"2021-06-03T08:45:20.281947Z","shell.execute_reply.started":"2021-06-03T08:42:27.732865Z","shell.execute_reply":"2021-06-03T08:45:20.280661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-03T08:45:20.283671Z","iopub.execute_input":"2021-06-03T08:45:20.284157Z","iopub.status.idle":"2021-06-03T08:45:20.318436Z","shell.execute_reply.started":"2021-06-03T08:45:20.284097Z","shell.execute_reply":"2021-06-03T08:45:20.31701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"Then we proceed to perform all our preprocessing steps:\n* lowercasing all text\n* removing all punctuation\n* removing stopwords\n* removing nonalphabetic tokens\n* removing words shorter than one character","metadata":{}},{"cell_type":"code","source":"# lowercasing the text\ntrain['text'] = train['text'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:45:20.321315Z","iopub.execute_input":"2021-06-03T08:45:20.32179Z","iopub.status.idle":"2021-06-03T08:45:24.883025Z","shell.execute_reply.started":"2021-06-03T08:45:20.32174Z","shell.execute_reply":"2021-06-03T08:45:24.882006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing punctuation\nre_punc = re.compile('[%s]' % re.escape(string.punctuation))\n\ndef remove_punctuation(text):\n    return ' '.join([re_punc.sub('', word) for word in str(text).split()])\n\ntrain['text'] = train['text'].apply(lambda text: remove_punctuation(text))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:45:24.8845Z","iopub.execute_input":"2021-06-03T08:45:24.88481Z","iopub.status.idle":"2021-06-03T08:47:02.685129Z","shell.execute_reply.started":"2021-06-03T08:45:24.884768Z","shell.execute_reply":"2021-06-03T08:47:02.684068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing stopwords\nstop_words = list(stopwords.words('english'))\n\ndef remove_stopwords(text):\n    return ' '.join([word for word in str(text).split() if word not in stop_words])\n\ntrain['text'] = train['text'].apply(lambda text: remove_stopwords(text))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:47:02.686954Z","iopub.execute_input":"2021-06-03T08:47:02.687399Z","iopub.status.idle":"2021-06-03T08:51:31.484016Z","shell.execute_reply.started":"2021-06-03T08:47:02.687349Z","shell.execute_reply":"2021-06-03T08:51:31.483087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing nonalphabetic tokens\ndef remove_nonalpha(text):\n    return ' '.join([word for word in str(text).split() if word.isalpha()])\n\ntrain['text'] = train['text'].apply(lambda text: remove_nonalpha(text))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:51:31.486587Z","iopub.execute_input":"2021-06-03T08:51:31.486903Z","iopub.status.idle":"2021-06-03T08:51:54.802614Z","shell.execute_reply.started":"2021-06-03T08:51:31.486871Z","shell.execute_reply":"2021-06-03T08:51:54.801771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing short words\ndef remove_short(text):\n    return ' '.join([word for word in str(text).split() if len(word) > 1])\n\ntrain['text'] = train['text'].apply(lambda text: remove_short(text))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:51:54.804284Z","iopub.execute_input":"2021-06-03T08:51:54.804722Z","iopub.status.idle":"2021-06-03T08:52:19.013537Z","shell.execute_reply.started":"2021-06-03T08:51:54.804688Z","shell.execute_reply":"2021-06-03T08:52:19.011914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# String matching and how to make predictions","metadata":{}},{"cell_type":"markdown","source":"We first get the list of the test files' names (we should have 4 of them). We will then create a similar dataframe to ``train.csv`` with only an ``Id`` column and the article's text.","metadata":{}},{"cell_type":"code","source":"test_filenames = os.listdir('../input/coleridgeinitiative-show-us-the-data/test')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:55:59.441833Z","iopub.execute_input":"2021-06-03T08:55:59.442218Z","iopub.status.idle":"2021-06-03T08:55:59.44748Z","shell.execute_reply.started":"2021-06-03T08:55:59.442185Z","shell.execute_reply":"2021-06-03T08:55:59.446285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# turn it into a dataframe: let's first get the Id column\ntest = pd.DataFrame({'Id':test_filenames})\ntest['Id'] = test['Id'].apply(lambda x : x.split('.')[0])\n\n# we now add the text column (with the article text)\ntest['text'] = test['Id'].apply(merge, test=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:56:01.272636Z","iopub.execute_input":"2021-06-03T08:56:01.273032Z","iopub.status.idle":"2021-06-03T08:56:01.333069Z","shell.execute_reply.started":"2021-06-03T08:56:01.272999Z","shell.execute_reply":"2021-06-03T08:56:01.332002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2021-06-03T08:56:04.074539Z","iopub.execute_input":"2021-06-03T08:56:04.074936Z","iopub.status.idle":"2021-06-03T08:56:04.090145Z","shell.execute_reply.started":"2021-06-03T08:56:04.074901Z","shell.execute_reply":"2021-06-03T08:56:04.088887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we must clean this text. But an easy way to have a first submission would be to search within the text column if we find dataset names that are already in our train set. We will perform that by keeping only words in the ``train[dataset_label]`` and ``train[dataset_title]`` columns (we refer to columns not cleaned because we have not cleaned the test articles' text).\n\nWe first extract the set of words from these 2 columns and define how we will clean our text (we keep alphanumeric characters, split the text into tokens, lowercase it and strip trailing and leading spaces). The set of title words consists of the union of the sets of (unique) words from the ``train[dataset_label]`` and ``train[dataset_title]`` columns, that we then lowercase.","metadata":{}},{"cell_type":"code","source":"title_words = [x.lower() for x in set(train['dataset_title'].unique()).union(set(train['dataset_label'].unique()))]\n\ndef clean_articles(text):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:17:58.378008Z","iopub.execute_input":"2021-06-03T09:17:58.378417Z","iopub.status.idle":"2021-06-03T09:17:58.399751Z","shell.execute_reply.started":"2021-06-03T09:17:58.378379Z","shell.execute_reply":"2021-06-03T09:17:58.398649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we load the submission template to fill.","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:15:49.430483Z","iopub.execute_input":"2021-06-03T09:15:49.430922Z","iopub.status.idle":"2021-06-03T09:15:49.437822Z","shell.execute_reply.started":"2021-06-03T09:15:49.430887Z","shell.execute_reply":"2021-06-03T09:15:49.436496Z"}}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:18:02.277293Z","iopub.execute_input":"2021-06-03T09:18:02.27773Z","iopub.status.idle":"2021-06-03T09:18:02.29604Z","shell.execute_reply.started":"2021-06-03T09:18:02.27769Z","shell.execute_reply":"2021-06-03T09:18:02.294732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we loop over the 4 rows of the submission dataframe to perform our string matching operation. More precisely, we get the 4 rows to concatenate the text of each article, and then we filter this text based on our ``title_words`` list of words. Finally we build a string a title words separated by ``|``.","metadata":{}},{"cell_type":"code","source":"article_titles = []\n\nfor idx in submission['Id']:\n    article_text = test[test['Id'] == idx].text.str.cat(sep = '\\n').lower()\n    label = []\n    \n    for data_title in title_words:\n        if data_title in article_text:\n            label.append(clean_articles(data_title))\n            \n    article_titles.append(\"|\".join(label))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:23:04.404127Z","iopub.execute_input":"2021-06-03T09:23:04.404524Z","iopub.status.idle":"2021-06-03T09:23:04.440978Z","shell.execute_reply.started":"2021-06-03T09:23:04.404493Z","shell.execute_reply":"2021-06-03T09:23:04.439894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's add this to our submission dataframe and check it.","metadata":{}},{"cell_type":"code","source":"submission['PredictionString'] = article_titles\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:24:24.369856Z","iopub.execute_input":"2021-06-03T09:24:24.370224Z","iopub.status.idle":"2021-06-03T09:24:24.382155Z","shell.execute_reply.started":"2021-06-03T09:24:24.370192Z","shell.execute_reply":"2021-06-03T09:24:24.381117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally we save our submission.","metadata":{}},{"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T09:24:44.536322Z","iopub.execute_input":"2021-06-03T09:24:44.536886Z","iopub.status.idle":"2021-06-03T09:24:44.555843Z","shell.execute_reply.started":"2021-06-03T09:24:44.536849Z","shell.execute_reply":"2021-06-03T09:24:44.554961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks for reading!\n\nThis notebook was inspired by the following resources:\n* https://neptune.ai/blog/exploratory-data-analysis-natural-language-processing-tools\n* https://www.kaggle.com/anthokalel/coleridge-complete-eda\n* https://www.kaggle.com/ajaypawar123/eda-text-processing-string-matching-beginners/notebook#String-Matching","metadata":{}}]}