{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import urllib.request, json \nimport pandas as pd\n#from pandas.io.json import json_normalize\n############## https://labs.data.gov/dashboard/offices/qa\napi_list = [\"http://www.usda.gov/data.json\",\"http://www.commerce.gov/data.json\",\"http://www.ed.gov/data.json\",\"https://healthdata.gov/data.json\",\"http://www.dol.gov/data.json\",\"https://www.doi.gov/sites/doi.gov/files/data.json\",\"http://www.energy.gov/data.json\",\"http://www.dhs.gov/data.json\",\"http://www.ustreas.gov/data.json\",\"http://www.dot.gov/data.json\",\"http://www.va.gov/data.json\",\"http://www.gsa.gov/data.json\",\"http://www.nsf.gov/data.json\",\"http://www.sba.gov/data.json\",\"http://www.ssa.gov/data.json\",\"http://www.usaid.gov/data.json\"]\n#api_list = [\"http://www.commerce.gov/data.json\"]\ni=0\nfor site in api_list:\n    print (site)\n    with urllib.request.urlopen(site) as url:\n        data = json.loads(url.read().decode())\n        #print (data)\n        if i==0:\n            df = pd.DataFrame(pd.json_normalize(data, 'dataset'))\n        else:\n            temp = pd.DataFrame(pd.json_normalize(data, 'dataset'))\n            df = df.append(temp, ignore_index=True)\n    i=i+1\n#df=df[df['@type'].str.contains('dcat:Dataset')]\ndf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-20T13:33:16.957905Z","iopub.execute_input":"2021-05-20T13:33:16.958796Z","iopub.status.idle":"2021-05-20T13:36:23.707599Z","shell.execute_reply.started":"2021-05-20T13:33:16.958749Z","shell.execute_reply":"2021-05-20T13:36:23.706748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = df[['title']].copy()\ndf2['title'].str.replace(\",\",\"\")\ndf2['title']=df['title'].str.lower()\ndf[\"title\"] = df['title'].str.replace('[^\\w\\s]','')\n\ndf2","metadata":{"execution":{"iopub.status.busy":"2021-05-20T13:36:41.455398Z","iopub.execute_input":"2021-05-20T13:36:41.455899Z","iopub.status.idle":"2021-05-20T13:36:42.520646Z","shell.execute_reply.started":"2021-05-20T13:36:41.455865Z","shell.execute_reply":"2021-05-20T13:36:42.519747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.to_csv('datasets.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T13:37:05.687039Z","iopub.execute_input":"2021-05-20T13:37:05.687415Z","iopub.status.idle":"2021-05-20T13:37:06.231202Z","shell.execute_reply.started":"2021-05-20T13:37:05.687386Z","shell.execute_reply":"2021-05-20T13:37:06.230175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df3=df[df['identifier'].str.match('noaa')]\n\ndf3=df2[df2['title'].str.contains('longitudinal')]\n\ndf3","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}