{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Creating training df for \"Coleridge - Show US the data\" competition","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport seaborn as sns\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"no_of_examples = None #reducing the number for experimentation, using None for final code (applied on papers dict generation only)\n\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\n\ntrain_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train[:no_of_examples]['Id'].unique():\n    with open(f'{train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating a dataframe with papers info and text divided into sections\n\ndf_list = []\n\nfor paper_id in train[:no_of_examples]['Id'].unique():\n    \n    for count, s in enumerate(papers[paper_id]):\n        extended_train = {}\n        \n        extended_train['Id'] = paper_id\n        extended_train['pub_title'] = train.loc[train.Id == paper_id, 'pub_title']\n        extended_train['dataset_title'] = train.loc[train.Id == paper_id, 'dataset_title']\n        extended_train['dataset_label'] = train.loc[train.Id == paper_id, 'dataset_label']\n        extended_train['cleaned_label'] = train.loc[train.Id == paper_id, 'cleaned_label']\n        \n        extended_train['section_title'] = s['section_title']\n        extended_train['section_number'] = count\n        extended_train['text'] = s['text']\n        \n        df_list.append(pd.DataFrame(extended_train))\n        \ndf_extended = pd.concat(df_list, ignore_index = True)\ndf_extended = df_extended.sort_values(['Id', 'section_number'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adding cleaned_text column\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndf_extended['cleaned_text'] = df_extended['text'].apply(clean_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adding a column indicating if a cleaned label is in the cleaned text with bool\ndf_extended['label_match'] = df_extended.apply(lambda x:x.cleaned_label in x.cleaned_text, axis =1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended.groupby('dataset_title').dataset_label.unique().apply(len)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended.groupby('dataset_title').cleaned_label.unique().apply(len)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_extended.to_pickle(\"coleridge_train_extended.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}