{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"print(\"Please upvote :)\")","metadata":{"id":"uxF62MWmjCMx","outputId":"778eee86-6aae-429b-cdf0-39e308c3517d","execution":{"iopub.status.busy":"2022-05-21T08:51:04.925414Z","iopub.execute_input":"2022-05-21T08:51:04.926197Z","iopub.status.idle":"2022-05-21T08:51:04.934559Z","shell.execute_reply.started":"2022-05-21T08:51:04.926146Z","shell.execute_reply":"2022-05-21T08:51:04.933522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Installations","metadata":{"id":"wXWp_Ri7jXfE"}},{"cell_type":"code","source":"pip install -q gitpython","metadata":{"id":"ENw-sEnejXJU","execution":{"iopub.status.busy":"2022-05-21T08:51:04.936391Z","iopub.execute_input":"2022-05-21T08:51:04.936636Z","iopub.status.idle":"2022-05-21T08:51:12.061589Z","shell.execute_reply.started":"2022-05-21T08:51:04.936608Z","shell.execute_reply":"2022-05-21T08:51:12.060218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{"id":"2ySMm6GnjCMy"}},{"cell_type":"code","source":"from tqdm import tqdm\nimport os\nimport shutil\nimport git\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom sklearn.utils import shuffle","metadata":{"id":"Cpnl405njCMz","execution":{"iopub.status.busy":"2022-05-21T08:51:12.063492Z","iopub.execute_input":"2022-05-21T08:51:12.064293Z","iopub.status.idle":"2022-05-21T08:51:12.070756Z","shell.execute_reply.started":"2022-05-21T08:51:12.064225Z","shell.execute_reply":"2022-05-21T08:51:12.069627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{"id":"Rtl-ILMljCMz"}},{"cell_type":"code","source":"i, n_urls = (\n    0,\n    10 ** 3,\n)  # i is in range(10) to select a tenth of repository urls; n_urls is the total number of repository urls\ni, n_urls","metadata":{"id":"x7xbHy1DjCMz","outputId":"0a1d8eeb-ef1f-4315-b087-3414cc2762fd","execution":{"iopub.status.busy":"2022-05-21T08:51:12.074097Z","iopub.execute_input":"2022-05-21T08:51:12.074908Z","iopub.status.idle":"2022-05-21T08:51:12.089151Z","shell.execute_reply.started":"2022-05-21T08:51:12.074864Z","shell.execute_reply":"2022-05-21T08:51:12.088164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{"id":"blk3gs3YjCM0"}},{"cell_type":"code","source":"def get_filepaths():\n    filepaths = []\n\n    for dirpath, _, filenames in os.walk(\"repository\"):\n        for filename in filenames:\n            if filename.endswith(\".py\"):  # Python files\n                filepath = os.path.join(dirpath, filename)\n\n                try:\n                    strng = open(filepath).read()  # read file\n\n                    if strng.strip() != \"\":  # non-empty strings\n                        filepaths.append(filepath)\n                except:\n                    pass\n\n    return filepaths","metadata":{"id":"jYXFWpNojCM0","execution":{"iopub.status.busy":"2022-05-21T08:51:12.090351Z","iopub.execute_input":"2022-05-21T08:51:12.091058Z","iopub.status.idle":"2022-05-21T08:51:12.102188Z","shell.execute_reply.started":"2022-05-21T08:51:12.091022Z","shell.execute_reply":"2022-05-21T08:51:12.101288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def move_python_files(j, filepaths):\n    for filepath in filepaths:\n        Path(filepath).rename(f\"scripts/script_{j}.py\")\n        j += 1\n    return j","metadata":{"id":"F9DhNuJ1jCM1","execution":{"iopub.status.busy":"2022-05-21T08:51:12.103351Z","iopub.execute_input":"2022-05-21T08:51:12.103719Z","iopub.status.idle":"2022-05-21T08:51:12.116981Z","shell.execute_reply.started":"2022-05-21T08:51:12.103691Z","shell.execute_reply":"2022-05-21T08:51:12.116195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Scripts","metadata":{"id":"5AO2KslNjCM1"}},{"cell_type":"code","source":"filepath = \"../input/urls-of-top-python-github-projects/urls_of_top_python_github_projects.csv\"\nurls = list(np.squeeze(pd.read_csv(filepath).values))[:n_urls]\nurls = shuffle(urls, random_state=0)\nurls[:5], len(urls)","metadata":{"id":"cGpdHiJvjCM2","outputId":"39739651-d65f-495f-a189-5073b366defb","execution":{"iopub.status.busy":"2022-05-21T08:51:12.118164Z","iopub.execute_input":"2022-05-21T08:51:12.118685Z","iopub.status.idle":"2022-05-21T08:51:12.187624Z","shell.execute_reply.started":"2022-05-21T08:51:12.11865Z","shell.execute_reply":"2022-05-21T08:51:12.18647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranges = np.linspace(0, len(urls), 11, dtype=int)\nurls = urls[ranges[i] : ranges[i + 1]]  # one tenth per notebook\nurls[:5], len(urls)","metadata":{"id":"lPHf_qwcjCM2","outputId":"3c890bb0-853c-49c4-ddb2-d02ecb9c007e","execution":{"iopub.status.busy":"2022-05-21T08:51:12.188915Z","iopub.execute_input":"2022-05-21T08:51:12.18913Z","iopub.status.idle":"2022-05-21T08:51:12.196176Z","shell.execute_reply.started":"2022-05-21T08:51:12.189107Z","shell.execute_reply":"2022-05-21T08:51:12.195345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"urls_for_removal = [  # problematic Python repositories\n    \"https://github.com/owid/covid-19-data.git\",\n    \"https://github.com/covid19india/api.git\",\n    \"https://github.com/echen102/COVID-19-TweetIDs.git\",\n]\n\nurls = [url for url in urls if url not in urls_for_removal]\nurls[:5], len(urls)","metadata":{"id":"8paHoPkWjCM2","outputId":"09b5ea8a-d49e-4225-9118-c848269385dc","execution":{"iopub.status.busy":"2022-05-21T08:51:12.200492Z","iopub.execute_input":"2022-05-21T08:51:12.200781Z","iopub.status.idle":"2022-05-21T08:51:12.214167Z","shell.execute_reply.started":"2022-05-21T08:51:12.200752Z","shell.execute_reply":"2022-05-21T08:51:12.213222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"j, directory = 0, \"scripts\"\n\nif os.path.isdir(directory):\n    shutil.rmtree(directory)\n\nos.makedirs(directory)\n\nif os.path.isdir(\"repository\"):\n    shutil.rmtree(\"repository\")\n\nfor url in tqdm(urls):\n    git.Repo.clone_from(url, \"repository\")\n    filepaths = get_filepaths()\n    j = move_python_files(j, filepaths)\n    shutil.rmtree(\"repository\")","metadata":{"id":"B9HbMntujCM2","outputId":"83d5942a-ff8c-42cc-f3c0-fd733118404c","execution":{"iopub.status.busy":"2022-05-21T08:51:12.215336Z","iopub.execute_input":"2022-05-21T08:51:12.216153Z","iopub.status.idle":"2022-05-21T08:52:21.175787Z","shell.execute_reply.started":"2022-05-21T08:51:12.216111Z","shell.execute_reply":"2022-05-21T08:52:21.174983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display","metadata":{}},{"cell_type":"code","source":"for filename in os.listdir(directory)[:2]:\n    filepath = os.path.join(directory, filename)\n    strng = open(filepath).read()\n    print(strng)\n    print(\n        \"*******************************************************************************************************\"\n    )","metadata":{"id":"a3RXpDznjCM2","outputId":"8f44514c-fc26-41e8-c8cb-2cf41e7e61a6","execution":{"iopub.status.busy":"2022-05-21T08:52:21.17809Z","iopub.execute_input":"2022-05-21T08:52:21.178481Z","iopub.status.idle":"2022-05-21T08:52:21.191874Z","shell.execute_reply.started":"2022-05-21T08:52:21.178424Z","shell.execute_reply":"2022-05-21T08:52:21.191172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save","metadata":{}},{"cell_type":"code","source":"!tar -czf scripts.tar.gz scripts","metadata":{"id":"tfx2LzfdjCM3","execution":{"iopub.status.busy":"2022-05-21T08:52:21.192847Z","iopub.execute_input":"2022-05-21T08:52:21.193589Z","iopub.status.idle":"2022-05-21T08:52:23.749358Z","shell.execute_reply.started":"2022-05-21T08:52:21.193553Z","shell.execute_reply":"2022-05-21T08:52:23.746952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree(\"scripts\")","metadata":{"id":"iYGBjiB8jCM3","execution":{"iopub.status.busy":"2022-05-21T08:52:23.751705Z","iopub.execute_input":"2022-05-21T08:52:23.752106Z","iopub.status.idle":"2022-05-21T08:52:23.953687Z","shell.execute_reply.started":"2022-05-21T08:52:23.752056Z","shell.execute_reply":"2022-05-21T08:52:23.952656Z"},"trusted":true},"execution_count":null,"outputs":[]}]}