{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"#The script will extract data from the https://kenpom.com/ for the years for which the data is available.\n#No data behind paywall can be accessed by this\nfrom bs4 import BeautifulSoup\nimport numpy as np\nimport pandas as pd\nimport requests\nimport re","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"urls = np.arange(2002, 2021, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for url in urls:\n    print(f\"http://kenpom.com/index.php?y={url}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"table = {'year' : [],\n        'rank': [],\n        'team': [],\n        'W-L': [],\n        'AdjEM':[],\n        'AdjO':[],\n        'AdjD':[],\n        'AdjT':[],\n        'Luck':[],\n        'SoSAdjEM':[],\n        'SoSOppO':[],\n        'SoSOppD':[]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tqdm import tqdm\nfor url in tqdm(urls):\n    r = requests.get(f\"http://kenpom.com/index.php?y={url}\")\n    soup = BeautifulSoup(r.text, \"lxml\")\n    for elem in soup.find_all('tbody'):\n        for row in elem.find_all('tr'):\n            try:            \n                table['rank'].append(int(row.find('td', class_='hard_left').text))\n                table['year'].append(url)\n                table['team'].append(str(row.find('td', class_='next_left').text))\n                table['W-L'].append(str(row.find('td', class_='wl').text))\n                table['AdjEM'].append(float(row.find('td', class_=None).text))\n                for i, row_elem in enumerate(row.find_all('td', class_='td-left divide')):\n                    if i==0:\n                        table['AdjO'].append(float(row_elem.text))\n                    elif i==1:\n                        table['AdjT'].append(float(row_elem.text))\n                    elif i==2:\n                        table['Luck'].append(float(row_elem.text))\n                    elif i==3:\n                        table['SoSAdjEM'].append(row_elem.text)\n                for i, row_elem in enumerate(row.find_all('td', class_='td-left')):\n                    if i== 1:\n                        table['AdjD'].append(float(row_elem.text))\n                    elif i==5:\n                        table['SoSOppO'].append(float(row_elem.text))\n                    elif i==6:\n                        table['SoSOppD'].append(float(row_elem.text))\n            except:\n                pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ken_pom_data = pd.DataFrame(table)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ken_pom_data['team'] = ken_pom_data.team.apply(lambda x: re.sub('[^a-zA-Z]+', '', x.lower()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#ken_pom_data.to_csv('Data/ken_pom.csv', index =False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"dl_cu100","language":"python","name":"dl_cu100"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.5"}},"nbformat":4,"nbformat_minor":1}