{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install bs4\n\n\nimport requests\nfrom bs4 import BeautifulSoup\n\nimport pandas as pd\nimport re\n\nimport time\nfrom IPython.display import clear_output\n\n\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:52:19.467185Z","iopub.execute_input":"2021-06-15T06:52:19.467613Z","iopub.status.idle":"2021-06-15T06:52:30.019114Z","shell.execute_reply.started":"2021-06-15T06:52:19.467507Z","shell.execute_reply":"2021-06-15T06:52:30.017987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"url_prefix = 'https://catalog.data.gov/dataset'\ntotal_data = 22000\npages = int(total_data / 20)\n\nprint(f'pages = {pages}')","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:52:30.02087Z","iopub.execute_input":"2021-06-15T06:52:30.021214Z","iopub.status.idle":"2021-06-15T06:52:30.026626Z","shell.execute_reply.started":"2021-06-15T06:52:30.021181Z","shell.execute_reply":"2021-06-15T06:52:30.025839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_docs_by_page(url):\n    r = requests.get(url)\n    time.sleep(3)\n    r.encoding = \"UTF-8\"\n\n    soup = BeautifulSoup(r.text,'html.parser')\n    dataset_blocks = soup.find_all(class_=\"dataset-heading\")\n\n    dataset_headings = []\n    for dataset_block in dataset_blocks:\n        dataset_heading = dataset_block.get_text()[1:-1]\n        dataset_headings.append(dataset_heading)\n\n    return dataset_headings\n\n\ndef get_all_pages(first_page, last_page, url):\n    data = []\n    print(f'page number from {first_page} to {(last_page)}')\n    for page_number in range(first_page, last_page+1):\n        print(f'page number: {page_number}', end='\\r')\n        url =  url_prefix + '?page=' + str(page_number)\n        docs = get_docs_by_page(url) \n        data.extend(docs)\n        \n    print(f'page number: {page_number}')\n    print(f'len(data) = {len(data)}')\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:52:30.027582Z","iopub.execute_input":"2021-06-15T06:52:30.027834Z","iopub.status.idle":"2021-06-15T06:52:30.040551Z","shell.execute_reply.started":"2021-06-15T06:52:30.027808Z","shell.execute_reply":"2021-06-15T06:52:30.03951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nadditional_gov_datasets_popular = get_all_pages(1, pages, url_prefix)\n\nadditional_gov_datasets_popular = pd.DataFrame(additional_gov_datasets_popular, columns=['title'])\nadditional_gov_datasets_popular.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:52:30.042275Z","iopub.execute_input":"2021-06-15T06:52:30.04259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"additional_gov_datasets_popular['title'] = additional_gov_datasets_popular['title'].apply(text_cleaning)\n\nadditional_gov_datasets_popular.to_csv(f'additional_gov_datasets_{total_data}popular.csv', index=False)\nadditional_gov_datasets_popular.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\nadnl_govt_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for final_row in range(2000, total_data+1, 2000):\n    data_set_800_with_popular = pd.concat([adnl_govt_labels, \n                                           additional_gov_datasets_popular.iloc[:final_row]]).reset_index(drop=True)\n    data_set_800_with_popular.to_csv(f'data_set_800_with{final_row}popular.csv', index=False)\n    print(f'data_set_800_with{final_row}popular.csv is saved with len = {len(data_set_800_with_popular)}')\n\ndata_set_800_with_popular","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}