{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"IN THIS NOTEBOOK I CREATE A READY TO GO DATASET. EACH ROW REPRESENT A DIFFERENT CHAPTER OF ALL THE PUBLICATION IN THE ORIGINAL DATAFRAME. IN THIS WAY EACH ROW IS FORMED BY THE NAME OF THE PUBLICATION, THE NAME OF THE DATASET(S) USED, THE LABEL(S), THE NAME OF THE CHAPTER AND THE TEXT OF THE CHAPTER. ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport json\nimport glob\nfrom tqdm.autonotebook import tqdm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_json(df = 'train'):\n    \n    if df == 'train':\n        path = '../input/coleridgeinitiative-show-us-the-data/train'\n        df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\n    elif df == 'test':\n        path = '../input/coleridgeinitiative-show-us-the-data/test'\n        df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\n    \n    \n    files = glob.glob(path + \"/*.json\")\n    \n    Ids = []\n    headings = []\n    content = []\n    publication = []\n    \n    for i in range(0,len(files)):\n        Ids.append(files[i][52:-5]) \n    \n    pt = []\n    dt = []\n    dl = []\n    cl = []\n\n    for i in df['Id'].unique():\n        pt.append((df[df['Id'] == i].pub_title).to_list())\n        dt.append((df[df['Id'] == i].dataset_title).to_list())\n        dl.append((df[df['Id'] == i].dataset_label).to_list())\n        cl.append((df[df['Id'] == i].cleaned_label).to_list())\n\n    df = pd.DataFrame({'Id':df['Id'].unique(),'pub_title':pt,'dataset_title':dt,'dataset_label':dl,'cleaned_label':cl})\n\n    \n    for j in range(0,len(files)):\n        with open(files[j],'r') as f:\n            json_decode = json.load(f)\n            for data in range(0,len(json_decode)):\n                headings.append(json_decode[data]['section_title'])\n                content.append(json_decode[data]['text'])\n                publication.append(Ids[j])\n    \n    contents = pd.DataFrame({'head': headings,'text': content, 'Id' : publication})       \n    \n    contents = pd.merge(contents,df, how = 'left', on = 'Id')\n    contents = contents.drop(columns=[\"Id\"])\n    contents = contents[['pub_title', 'dataset_title', 'dataset_label','cleaned_label', 'head', 'text']]\n    \n    return(contents)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nread_json('train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}