{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install bs4\n\n\nimport requests\nfrom bs4 import BeautifulSoup\n\nimport pandas as pd\nimport re\n\nimport time\nfrom IPython.display import clear_output\n\n\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T04:16:50.492497Z","iopub.execute_input":"2021-05-25T04:16:50.492958Z","iopub.status.idle":"2021-05-25T04:16:57.128131Z","shell.execute_reply.started":"2021-05-25T04:16:50.492918Z","shell.execute_reply":"2021-05-25T04:16:57.126821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"url_prefix = 'https://catalog.data.gov/dataset'\ntotal_data = 22000\npages = int(total_data / 20)\n\nprint(f'pages = {pages}')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:31:23.709715Z","iopub.execute_input":"2021-05-23T08:31:23.710511Z","iopub.status.idle":"2021-05-23T08:31:23.716485Z","shell.execute_reply.started":"2021-05-23T08:31:23.710445Z","shell.execute_reply":"2021-05-23T08:31:23.71536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_docs_by_page(url):\n    r = requests.get(url)\n    time.sleep(3)\n    r.encoding = \"UTF-8\"\n\n    soup = BeautifulSoup(r.text,'html.parser')\n    dataset_blocks = soup.find_all(class_=\"dataset-heading\")\n\n    dataset_headings = []\n    for dataset_block in dataset_blocks:\n        dataset_heading = dataset_block.get_text()[1:-1]\n        dataset_headings.append(dataset_heading)\n\n    return dataset_headings\n\n\ndef get_all_pages(first_page, last_page, url):\n    data = []\n    print(f'page number from {first_page} to {(last_page)}')\n    for page_number in range(first_page, last_page+1):\n        print(f'page number: {page_number}', end='\\r')\n        url =  url_prefix + '?page=' + str(page_number)\n        docs = get_docs_by_page(url) \n        data.extend(docs)\n        \n    print(f'page number: {page_number}')\n    print(f'len(data) = {len(data)}')\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:01:56.453193Z","iopub.execute_input":"2021-05-23T08:01:56.453545Z","iopub.status.idle":"2021-05-23T08:01:56.465924Z","shell.execute_reply.started":"2021-05-23T08:01:56.453502Z","shell.execute_reply":"2021-05-23T08:01:56.464923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nadditional_gov_datasets_popular = get_all_pages(1, pages, url_prefix)\n\nadditional_gov_datasets_popular = pd.DataFrame(additional_gov_datasets_popular, columns=['title'])\nadditional_gov_datasets_popular.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:01:56.467432Z","iopub.execute_input":"2021-05-23T08:01:56.467796Z","iopub.status.idle":"2021-05-23T08:09:23.223462Z","shell.execute_reply.started":"2021-05-23T08:01:56.467764Z","shell.execute_reply":"2021-05-23T08:09:23.221787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-05-25T04:16:57.130609Z","iopub.execute_input":"2021-05-25T04:16:57.131394Z","iopub.status.idle":"2021-05-25T04:16:57.138241Z","shell.execute_reply.started":"2021-05-25T04:16:57.131344Z","shell.execute_reply":"2021-05-25T04:16:57.137333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"additional_gov_datasets_popular['title'] = additional_gov_datasets_popular['title'].apply(text_cleaning)\n\nadditional_gov_datasets_popular.to_csv(f'additional_gov_datasets_{total_data}popular.csv', index=False)\nadditional_gov_datasets_popular.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:16:09.986481Z","iopub.execute_input":"2021-05-23T08:16:09.987069Z","iopub.status.idle":"2021-05-23T08:16:10.034954Z","shell.execute_reply.started":"2021-05-23T08:16:09.987034Z","shell.execute_reply":"2021-05-23T08:16:10.03409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\nadnl_govt_labels.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:17:23.636079Z","iopub.execute_input":"2021-05-23T08:17:23.636599Z","iopub.status.idle":"2021-05-23T08:17:23.668544Z","shell.execute_reply.started":"2021-05-23T08:17:23.636568Z","shell.execute_reply":"2021-05-23T08:17:23.667163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for final_row in range(2000, total_data+1, 2000):\n    data_set_800_with_popular = pd.concat([adnl_govt_labels, \n                                           additional_gov_datasets_popular.iloc[:final_row]]).reset_index(drop=True)\n    data_set_800_with_popular.to_csv(f'data_set_800_with{final_row}popular.csv', index=False)\n    print(f'data_set_800_with{final_row}popular.csv is saved with len = {len(data_set_800_with_popular)}')\n\ndata_set_800_with_popular","metadata":{"execution":{"iopub.status.busy":"2021-05-23T08:35:25.849146Z","iopub.execute_input":"2021-05-23T08:35:25.84953Z","iopub.status.idle":"2021-05-23T08:35:25.859326Z","shell.execute_reply.started":"2021-05-23T08:35:25.849496Z","shell.execute_reply":"2021-05-23T08:35:25.85753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}