{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nimport string\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:32:04.563863Z","iopub.execute_input":"2023-05-16T11:32:04.564651Z","iopub.status.idle":"2023-05-16T11:32:04.572273Z","shell.execute_reply.started":"2023-05-16T11:32:04.564596Z","shell.execute_reply":"2023-05-16T11:32:04.571083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_random_string():\n    chars = string.ascii_lowercase + string.digits\n    return ''.join(random.choice(chars) for _ in range(8))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_PAGES = 10000","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def traverse_wiki(url, depth=2, visited=None, result=None):\n    if visited is None:\n        visited = set()\n    if result is None:\n        result = dict()\n    \n    # Stop if the depth is less than 0 or if we've visited more than MAX_PAGES pages\n    if url in visited or depth < 0 or len(visited) > MAX_PAGES:\n        return result\n\n    visited.add(url)\n\n    try:\n        print(f\"Page [{len(visited)+1}]: {url}\")\n        response = requests.get(url)\n        soup = BeautifulSoup(response.text, 'html.parser')\n        hyperlink_words = []\n\n        # Look for links in different parts of the page\n        for tag in soup.find_all(['dt', 'div', 'table']):\n            if (tag.name == 'div' and tag.get('id') == 'mw-content-text') or \\\n                (tag.name == 'table' and 'wikitable' in tag.get('class', [])) or \\\n                tag.name == 'dt':\n                    for link in tag.find_all('a', href=True):\n                        hyperlink_words.append(link.text)\n\n                        # Only follow links to other Wikipedia pages\n                        if link['href'].startswith('/wiki/') and ':' not in link['href']:\n                            new_url = 'https://en.wikipedia.org' + link['href']\n                            traverse_wiki(new_url, depth=depth-1, visited=visited, result=result)\n\n        result[url] = hyperlink_words\n    except Exception as e:\n        print(f\"Error while processing {url}: {e}\")\n    \n    return result\n","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:32:54.106415Z","iopub.execute_input":"2023-05-16T11:32:54.106862Z","iopub.status.idle":"2023-05-16T11:32:54.118858Z","shell.execute_reply.started":"2023-05-16T11:32:54.106822Z","shell.execute_reply":"2023-05-16T11:32:54.117644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# URLS = [\n#     \"https://en.wikipedia.org/wiki/Branches_of_physics\",   \n#     \"https://en.wikipedia.org/wiki/Glossary_of_chemistry_terms\",\n#     \"https://en.wikipedia.org/wiki/Index_of_chemistry_articles\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_areas_of_mathematics\",\n#     \"https://en.wikipedia.org/wiki/Lists_of_mathematics_topics\",\n#     \"https://en.wikipedia.org/wiki/Outline_of_mathematics\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_physics\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_biology\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_civil_engineering\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_electrical_and_electronics_engineering\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_mechanical_engineering\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_structural_engineering\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_aerospace_engineering\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_artificial_intelligence\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_astronomy\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_probability_and_statistics\",\n# ]\n\n# URLS = [\n#     \"https://en.wikipedia.org/wiki/Glossary_of_geography_terms\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_computer_science\",\n#     \"https://en.wikipedia.org/wiki/List_of_computing_and_IT_abbreviations\",\n#     \"https://en.wikipedia.org/wiki/Category:Computing_terminology\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_shapes_with_metaphorical_names\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_tensor_theory\",\n#     \"https://en.wikipedia.org/wiki/Glossary_of_philosophy\",\n#     \"https://en.wikipedia.org/wiki/Outline_of_metalworking\",\n#     \"https://en.wikipedia.org/wiki/Wikipedia:Contents/Health_and_fitness\",\n#     \"https://en.wikipedia.org/wiki/Wikipedia:Contents/Technology_and_applied_sciences\",\n# ]\n\n# URLS = [\n#     \"https://en.wikipedia.org/wiki/Lists_of_abbreviations\",\n#     \"https://en.wikipedia.org/wiki/List_of_common_misconceptions\",\n#     \"https://en.wikipedia.org/wiki/Timeline_of_musical_events\",\n#     \"https://en.wikipedia.org/wiki/Television_broadcasting\",\n#     \"https://en.wikipedia.org/wiki/Lists_of_earthquakes\",\n#     \"https://en.wikipedia.org/wiki/List_of_minerals\",\n#     \"https://en.wikipedia.org/wiki/List_of_rock_types\",\n#     \"https://en.wikipedia.org/wiki/List_of_letters_used_in_mathematics_and_science\",\n#     \"https://en.wikipedia.org/wiki/Wikipedia:Contents/Society_and_social_sciences\",\n#     \"https://en.wikipedia.org/wiki/Index_of_fractal-related_articles\",\n# ]\n\nURLS = [\n#     \"https://en.wikipedia.org/wiki/List_of_search_engines\",\n    \"https://en.wikipedia.org/wiki/Wikipedia:Good_articles/Natural_sciences\"\n]\nlen(URLS)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 0 # 0\nresult = traverse_wiki(URLS[idx])","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:39:44.006887Z","iopub.execute_input":"2023-05-16T11:39:44.007552Z","iopub.status.idle":"2023-05-16T11:39:50.765948Z","shell.execute_reply.started":"2023-05-16T11:39:44.007519Z","shell.execute_reply":"2023-05-16T11:39:50.764240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_result = dict()\nfor k, v in result.items():\n    new_k = k.split(\"/\")[-1]\n    new_v = list(set([elem for elem in v if (len(elem)>=4)&(not elem.startswith('http'))]))\n    processed_result[new_k] = new_v","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:33:11.318650Z","iopub.execute_input":"2023-05-16T11:33:11.319426Z","iopub.status.idle":"2023-05-16T11:33:11.338155Z","shell.execute_reply.started":"2023-05-16T11:33:11.319382Z","shell.execute_reply":"2023-05-16T11:33:11.336569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed_result['Kemistry']","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:33:12.763807Z","iopub.execute_input":"2023-05-16T11:33:12.764209Z","iopub.status.idle":"2023-05-16T11:33:12.769457Z","shell.execute_reply.started":"2023-05-16T11:33:12.764179Z","shell.execute_reply":"2023-05-16T11:33:12.768275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.Series(processed_result).reset_index()\ndf.columns = ['title', 'keywords']\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:33:14.502746Z","iopub.execute_input":"2023-05-16T11:33:14.503724Z","iopub.status.idle":"2023-05-16T11:33:14.552371Z","shell.execute_reply.started":"2023-05-16T11:33:14.503682Z","shell.execute_reply":"2023-05-16T11:33:14.551241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uid = generate_random_string()\n\ntry:\n    df.to_pickle(f'keywords_wiki_{uid}.pickle')\nexcept Exception as e:\n    print(e)\n\ntry:\n    df.to_csv(f'keywords_wiki_{uid}.csv', index=False)\nexcept Exception as e:\n    print(e)\n\ntry:\n    df.to_parquet(f'keywords_wiki_{uid}.parquet')\nexcept Exception as e:\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T11:33:55.529395Z","iopub.execute_input":"2023-05-16T11:33:55.529834Z","iopub.status.idle":"2023-05-16T11:33:55.801619Z","shell.execute_reply.started":"2023-05-16T11:33:55.529801Z","shell.execute_reply":"2023-05-16T11:33:55.800595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}