{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Data import**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dir_prefix = '/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"teams = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MTeams.csv\")\nseasons = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MSeasons.csv\")\ntourney_seeds = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MNCAATourneySeeds.csv\")\nreg_season_result = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MRegularSeasonCompactResults.csv\")\ntournament_result = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MNCAATourneyCompactResults.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_season_gbg = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MRegularSeasonDetailedResults.csv\")\ntournament_gbg = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MNCAATourneyDetailedResults.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cities = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/Cities.csv\")\ngame_cities = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MGameCities.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rankings = pd.read_csv(dir_prefix + \"MDataFiles_Stage1/MMasseyOrdinals.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Methods**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCalculate offensive and defensive efficiency of a team throughout the season\n\"\"\"\ndef calcEfficiency(dataset,teamID):\n game_won = dataset.loc[dataset['WTeamID'] == teamID]\n game_lost = dataset.loc[dataset['LTeamID'] == teamID]\n total_score = 0\n total_score_allowed = 0\n total_FGA = 0\n total_OR = 0\n total_TO = 0\n total_FTA = 0\n for index, row in game_won.iterrows():\n total_score = total_score + row['WScore']\n total_score_allowed = total_score_allowed + row['LScore']\n total_FGA = total_FGA + row['WFGA']\n total_OR = total_OR + row['WOR']\n total_TO = total_TO + row['WTO']\n total_FTA = total_FTA + row['WFTA']\n for index, row in game_lost.iterrows():\n total_score = total_score + row['LScore']\n total_score_allowed = total_score_allowed + row['WScore']\n total_FGA = total_FGA + row['LFGA']\n total_OR = total_OR + row['LOR']\n total_TO = total_TO + row['LTO']\n total_FTA = total_FTA + row['LFTA']\n poss = total_FGA - total_OR + total_TO + 0.4 * total_FTA\n if teamID != 1309:\n oeff = total_score / poss\n deff = total_score_allowed / poss\n return oeff, deff\n else:\n return 0,0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCalculate efficiency for all teams in a single season\n\"\"\"\ndef applyEff_team(dataset, year):\n OEff = []\n DEff = []\n for team, lastYear, firstYear in zip(teams['TeamID'],teams['LastD1Season'], teams['FirstD1Season']):\n if (firstYear <= year) and (lastYear >= year):\n OEff_score, DEff_score = calcEfficiency(dataset,team)\n OEff.append(OEff_score)\n DEff.append(DEff_score)\n else:\n OEff.append('NaN')\n DEff.append('NaN')\n return OEff, DEff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nThis function create ID and label for each game by default\nOptional parameters to calculate Seed_diff and ranking_diff is available\n\"\"\"\ndef featureAddition(dataset,seed=None, rank=None, elo=None):\n for index, row in dataset.iterrows():\n lower_ID = row['WTeamID']\n higher_ID = row['LTeamID']\n if seed != None:\n seed_diff = int(row['WTeamSeedPure']) - int(row['LTeamSeedPure'])\n if rank != None:\n ranking_diff = int(row['WRanking']) - int(row['LRanking'])\n if elo != None:\n lower_elo = row['w_elo']\n higher_elo = row['l_elo']\n\n if lower_ID > higher_ID:\n tmp = lower_ID\n lower_ID = higher_ID\n higher_ID = tmp\n if elo != None:\n tmp_elo = lower_elo\n lower_elo = higher_elo\n higher_elo = tmp_elo\n if rank != None:\n ranking_diff = int(row['LRanking']) - int(row['WRanking'])\n if seed != None:\n seed_diff = int(row['LTeamSeedPure']) - int(row['WTeamSeedPure'])\n #Id\n dataset.loc[index, 'ID'] = (str(row['Season']) + \"_\" + str(lower_ID) + \"_\" + str(higher_ID))\n #Label\n if lower_ID == row['WTeamID']:\n dataset.loc[index,'lower_win'] = 1\n else:\n dataset.loc[index,'lower_win'] = 0\n #Seed diff\n if seed != None:\n dataset.loc[index, 'Seed_diff'] = seed_diff\n #Ranking diff\n if rank != None:\n dataset.loc[index, 'Ranking_diff'] = ranking_diff\n if elo != None:\n dataset.loc[index, 'lower_elo'] = lower_elo\n dataset.loc[index,'higher_elo'] = higher_elo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nApply efficiency of teams into a single season dataset\n\"\"\"\ndef applyEff_season(dataset,year=None):\n for index, row in dataset.iterrows():\n if year == None:\n season = row['Season']\n else:\n season = year\n firstT, secondT = getTeam(dataset, index)\n dataset.loc[index,'Team1_OEff'] = round(float(teams[teams['TeamID'] == int(firstT)]['Offensive_Eff_' + str(season)].values) * 100,1)\n dataset.loc[index,'Team1_DEff'] = round(float(teams[teams['TeamID'] == int(firstT)]['Defensive_Eff_' + str(season)].values) * 100,1)\n dataset.loc[index,'Team2_OEff'] = round(float(teams[teams['TeamID'] == int(secondT)]['Offensive_Eff_' + str(season)].values) * 100,1)\n dataset.loc[index,'Team2_DEff'] = round(float(teams[teams['TeamID'] == int(secondT)]['Defensive_Eff_' + str(season)].values) * 100,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nSeparate seed into separate seasons\n\"\"\"\ndef seedSeparation(years):\n tournament_seeds = []\n for year in years:\n tournament_seeds.append(tourney_seeds.groupby(['Season']).get_group(year))\n return tournament_seeds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nGet teamID based on gameID\n\"\"\"\ndef getTeam(dataset,index):\n season = dataset['ID'][index][0:4]\n firstT = dataset['ID'][index][5:9]\n secondT = dataset['ID'][index][10:14]\n return season, firstT, secondT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def appendPred(df):\n pred = voting_clf.predict_proba(df[['log_lower_elo','log_higher_elo']])\n df['Pred'] = pred[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generateGbgSeparation(years):\n reg_gbg = []\n for year in years:\n reg_gbg.append(reg_season_gbg[reg_season_gbg['Season'] == year])\n return reg_gbg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nSeparate regular season and tournament game results into separate seasons\n\"\"\"\ndef yearSeparation(years):\n reg_season_sep = []\n tournament_sep = []\n for year in years:\n reg_season_sep.append(reg_season_result.loc[reg_season_result['Season'] == year])\n tournament_sep.append(tournament_result.loc[tournament_result['Season'] == year])\n #Sort tournament games by DayNum\n for i in range(len(years)):\n tournament_sep[i] = tournament_sep[i].sort_values('DayNum')\n return reg_season_sep, tournament_sep","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nBin elo score\n\"\"\"\ndef elo_bin(df, attr, bins=3):\n df['tmp'] = pd.cut(df[attr],bins)\n int_len = len(df['tmp'].unique())\n for itv in range(int_len):\n left_bound = (df['tmp'].unique()[int_len - 1 -itv]).left\n right_bound = (df['tmp'].unique()[int_len - 1 - itv]).right\n curr_interval = pd.Interval(left=left_bound, right=right_bound)\n df.loc[(df[attr] == curr_interval), attr] = itv + 1\n\n df[attr] = df[attr].astype(int)\n df.drop('tmp', axis=1, inplace=True)\n new_df = pd.get_dummies(df, columns=[attr], prefix=attr)\n return new_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calcRanking(df, ranking_df):\n for index, row in df.iterrows():\n season, first, second = getTeam(df, index)\n lower_ranking = ranking_df[(ranking_df['Season'] == int(season)) & (ranking_df['TeamID'] == int(first))]['Avg_rank'].values[0]\n higher_ranking = ranking_df[(ranking_df['Season'] == int(season)) & (ranking_df['TeamID'] == int(second))]['Avg_rank'].values[0]\n df.loc[index, 'lower_ranking'] = lower_ranking\n df.loc[index, 'higher_ranking'] = higher_ranking\n print('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rankingSysInfo(rankingSys,teamID):\n day_eval = {}\n team_rank = {}\n groupByTeam = rankings.loc[(rankings['SystemName'] == rankingSys)&(rankings['TeamID'] == teamID)].sort_values('RankingDayNum').groupby(['Season'])\n for name, group in groupByTeam:\n team_rank[name-2014] = group['OrdinalRank']\n day_eval[name-2014] = group['RankingDayNum']\n return day_eval, team_rank","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotAcrossSeasons(season,teamID):\n plt.figure(figsize=(20,20))\n for rankingSys in (rankings['SystemName'].unique()):\n if len(rankings.loc[(rankings['SystemName'] == rankingSys)&(rankings['TeamID'] == teamID)&(rankings['Season'] == season)]) != 0:\n day_eval, team_rank = rankingSysInfo(rankingSys,teamID)\n single_season_trend = pd.DataFrame({\"Day\": day_eval[season-2014], \"Rank\": team_rank[season-2014]})\n plt.plot(single_season_trend[\"Day\"],single_season_trend[\"Rank\"],label=rankingSys)\n plt.title(season)\n plt.xlabel('Days')\n plt.ylabel('Rank')\n plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gameSum(season, teamid):\n gbg = reg_gbg[season - 2010]\n game_played = gbg[(gbg['WTeamID'] == teamid) | (gbg['LTeamID'] == teamid)]\n num_game = len(game_played)\n score = 0\n off_reb = 0\n def_reb = 0\n turnover = 0\n assist = 0\n #General defense ability\n steal = 0\n block = 0\n #Demonstrate perimeter offense\n fgm3 = 0\n fga3 = 0\n #Demonstrate overall offense\n fgm = 0\n fga = 0\n #Demonstrate ability to draw a foul\n fta = 0\n #Demonstrate ability to control foul trouble\n pf = 0\n #Shooting percentage\n pct3 = 0\n pct = 0\n \n for index, row in game_played.iterrows():\n if row['LTeamID'] == teamid:\n score = score + row['LScore']\n off_reb = off_reb + row['LOR']\n def_reb = def_reb + row['LDR']\n turnover = turnover + row['LTO']\n assist = assist + row['LAst']\n steal = steal + row['LStl']\n block = block + row['LBlk']\n fgm3 = fgm3 + row['LFGM3']\n fga3 = fga3 + row['LFGA3']\n fgm = fgm + row['LFGM']\n fga = fga + row['LFGA']\n pf = pf + row['LPF']\n if row['WTeamID'] == teamid:\n score = score + row['WScore']\n off_reb = off_reb + row['WOR']\n def_reb = def_reb + row['WDR']\n turnover = turnover + row['WTO']\n assist = assist + row['WAst']\n steal = steal + row['WStl']\n block = block + row['WBlk']\n fgm3 = fgm3 + row['WFGM3']\n fga3 = fga3 + row['WFGA3']\n fgm = fgm + row['WFGM']\n fga = fga + row['WFGA']\n pf = pf + row['WPF']\n pct3 = fgm3/ fga3 * 100\n \n return score/num_game, off_reb/num_game, def_reb/num_game, turnover/num_game, assist/num_game, steal/num_game, block/num_game, fga3/num_game, pct3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

OEff-DEff

","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#This patitions the regular season dataset by year from 2010 to 2019\nreg_gbg = generateGbgSeparation(range(2010,2020))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This is season 2010\nreg_gbg[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Unfortunately this Offensive and defensive efficiency approach on my first try doesn't work very well regards of the public LB so I commented it out.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# #This calculates the offensive and defensive efficiency of each team in each regular season and apply the result to the 'team' dataset\n# for reg, year in zip(reg_gbg,range(2010,2020)):\n# OEff, DEff = applyEff_team(reg, year)\n# teams['Offensive_Eff_' + str(year)] = OEff\n# teams['Defensive_Eff_' + str(year)] = DEff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for reg, year in zip([reg_2015, reg_2016, reg_2017, reg_2018, reg_2019], range(2015,2020)):\n# applyEff_season(reg)\n# print(str(year) + 'done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for tourney, year in zip([tourney_2015, tourney_2016, tourney_2017, tourney_2018, tourney_2019], range(2015,2020)):\n# applyEff_season(tourney, year)\n# print(str(year) + 'done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for train in [reg_2015, reg_2016, reg_2017, reg_2018, reg_2019, tourney_2015, tourney_2016, tourney_2017, tourney_2018, tourney_2019]:\n# train['OEff_diff'] = train['Team1_OEff'] - train['Team2_OEff']\n# train['DEff_diff'] = train['Team1_DEff'] - train['Team2_DEff']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

Elo Score counting

","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_season_from_10 = reg_season_gbg.loc[reg_season_gbg['Season'] >= 2010]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_season_from_10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_elo = 1600\nelo_mess = reg_season_result\nteam_ids = set(elo_mess.WTeamID).union(set(elo_mess.LTeamID))\nelo_dict = dict(zip(list(team_ids), [1500] * len(team_ids)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#These value are able to tuned for a potential better performance\nK = 20\nHOME_ADVANTAGE = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This calculate the margin of victory\nelo_mess['margin'] = elo_mess.WScore - elo_mess.LScore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def elo_pred(elo1, elo2):\n return(1. / (10. ** (-(elo1 - elo2) / 400.) + 1.))\n\ndef expected_margin(elo_diff):\n return((7.5 + 0.006 * elo_diff))\n\ndef elo_update(w_elo, l_elo, margin):\n elo_diff = w_elo - l_elo\n pred = elo_pred(w_elo, l_elo)\n mult = ((margin + 3.) ** 0.8) / expected_margin(elo_diff)\n update = K * mult * (1 - pred)\n return(pred, update)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nw_elo = []\nl_elo = []\n\n# Loop over all rows of the games dataframe\nfor row in elo_mess.itertuples():\n \n # Get key data from current row\n w = row.WTeamID\n l = row.LTeamID\n margin = row.margin\n wloc = row.WLoc\n \n # Does either team get a home-court advantage?\n w_ad, l_ad, = 0., 0.\n if wloc == \"H\":\n w_ad += HOME_ADVANTAGE\n elif wloc == \"A\":\n l_ad += HOME_ADVANTAGE\n \n # Get elo updates as a result of the game\n pred, update = elo_update(elo_dict[w] + w_ad,\n elo_dict[l] + l_ad, \n margin)\n elo_dict[w] += update\n elo_dict[l] -= update\n \n # Save prediction and new Elos for each round\n preds.append(pred)\n w_elo.append(elo_dict[w])\n l_elo.append(elo_dict[l])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def final_elo_per_season(df, team_id):\n d = df.copy()\n d = d.loc[(d.WTeamID == team_id) | (d.LTeamID == team_id), :]\n d.sort_values(['Season', 'DayNum'], inplace=True)\n d.drop_duplicates(['Season'], keep='last', inplace=True)\n w_mask = d.WTeamID == team_id\n l_mask = d.LTeamID == team_id\n d['season_elo'] = None\n d.loc[w_mask, 'season_elo'] = d.loc[w_mask, 'w_elo']\n d.loc[l_mask, 'season_elo'] = d.loc[l_mask, 'l_elo']\n out = pd.DataFrame({\n 'team_id': team_id,\n 'Season': d.Season,\n 'season_elo': d.season_elo\n })\n return(out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"elo_mess['w_elo'] = w_elo\nelo_mess['l_elo'] = l_elo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_list = [final_elo_per_season(elo_mess, id) for id in team_ids]\nseason_elos = pd.concat(df_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_elos[season_elos['Season'] == 2019]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"reg_2019 = elo_mess[(elo_mess['Season'] >= 2015) & (elo_mess['Season'] <= 2019)].sort_values('DayNum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_2018 = elo_mess[(elo_mess['Season'] >= 2014) & (elo_mess['Season'] <= 2018)].sort_values('DayNum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_2017 = elo_mess[(elo_mess['Season'] >= 2013) & (elo_mess['Season'] <= 2017)].sort_values('DayNum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_2016 = elo_mess[(elo_mess['Season'] >= 2012) & (elo_mess['Season'] <= 2016)].sort_values('DayNum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_2015 = elo_mess[(elo_mess['Season'] >= 2011) & (elo_mess['Season'] <= 2015)].sort_values('DayNum')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

Last-day-ranking

","execution_count":null},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"last_day_ranking = rankings[rankings['RankingDayNum'] == 133]\nlast_day_ranking","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2015 = last_day_ranking[(last_day_ranking['Season'] <= 2015) & (last_day_ranking['Season'] >= 2011)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2016 = last_day_ranking[(last_day_ranking['Season'] <= 2016) & (last_day_ranking['Season'] >= 2012)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2017 = last_day_ranking[(last_day_ranking['Season'] <= 2017) & (last_day_ranking['Season'] >= 2013)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2018 = last_day_ranking[(last_day_ranking['Season'] <= 2018) & (last_day_ranking['Season'] >= 2014)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2019 = last_day_ranking[(last_day_ranking['Season'] <= 2019) & (last_day_ranking['Season'] >= 2015)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ranking_helper(src,dest):\n for name, group in src:\n curr_season = name[0]\n curr_team = name[1]\n avg = round(np.sum(group['OrdinalRank']) / len(group['OrdinalRank']),1)\n dest = dest.append({'Season':int(curr_season), 'TeamID':int(curr_team), 'Avg_rank':int(avg)}, ignore_index=True)\n return dest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2019_group = ranking_2019.groupby(['Season','TeamID'])\nlast_day_ranking_2019 = pd.DataFrame(columns=['Season', 'TeamID', 'Avg_rank'])\nlast_day_ranking_2019 = ranking_helper(ranking_2019_group, last_day_ranking_2019)\nlast_day_ranking_2019","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2018_group = ranking_2018.groupby(['Season','TeamID'])\nlast_day_ranking_2018 = pd.DataFrame(columns=['Season', 'TeamID', 'Avg_rank'])\nlast_day_ranking_2018 = ranking_helper(ranking_2018_group, last_day_ranking_2018)\nlast_day_ranking_2018","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2017_group = ranking_2017.groupby(['Season','TeamID'])\nlast_day_ranking_2017 = pd.DataFrame(columns=['Season', 'TeamID', 'Avg_rank'])\nlast_day_ranking_2017 = ranking_helper(ranking_2017_group, last_day_ranking_2017)\nlast_day_ranking_2017","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2016_group = ranking_2016.groupby(['Season','TeamID'])\nlast_day_ranking_2016 = pd.DataFrame(columns=['Season', 'TeamID', 'Avg_rank'])\nlast_day_ranking_2016 = ranking_helper(ranking_2016_group, last_day_ranking_2016)\nlast_day_ranking_2016","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ranking_2015_group = ranking_2015.groupby(['Season','TeamID'])\nlast_day_ranking_2015 = pd.DataFrame(columns=['Season', 'TeamID', 'Avg_rank'])\nlast_day_ranking_2015 = ranking_helper(ranking_2015_group, last_day_ranking_2015)\nlast_day_ranking_2015","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

Test modeling

","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import cross_val_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_2019 = tournament_result[(tournament_result['Season'] <= 2019) & (tournament_result['Season'] >= 2015)]\nelo_filtered = season_elos[(season_elos['Season'] <= 2019) & (season_elos['Season'] >= 2015)]\na = tourney_2019.merge(elo_filtered, left_on=['WTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'w_elo'})\ntourney_2019 = a.merge(elo_filtered, left_on=['LTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'l_elo'})\n#Usable output\nval_2019 = tourney_2019[tourney_2019['Season'] == 2019]\ntourney_2019 = tourney_2019[(tourney_2019['Season'] < 2019) & (tourney_2019['Season'] >= 2015)]\ntest_2019 = tourney_2019[tourney_2019['Season'] == 2018]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_2018 = tournament_result[(tournament_result['Season'] <= 2018) & (tournament_result['Season'] >= 2014)]\nelo_filtered = season_elos[(season_elos['Season'] <= 2018) & (season_elos['Season'] >= 2014)]\na = tourney_2018.merge(elo_filtered, left_on=['WTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'w_elo'})\ntourney_2018 = a.merge(elo_filtered, left_on=['LTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'l_elo'})\n#Usable output\nval_2018 = tourney_2018[tourney_2018['Season'] == 2018]\ntourney_2018 = tourney_2018[(tourney_2018['Season'] < 2018) & (tourney_2018['Season'] >= 2014)]\ntest_2018 = tourney_2018[tourney_2018['Season'] == 2017]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_2017 = tournament_result[(tournament_result['Season'] <= 2017) & (tournament_result['Season'] >= 2013)]\nelo_filtered = season_elos[(season_elos['Season'] <= 2017) & (season_elos['Season'] >= 2013)]\na = tourney_2017.merge(elo_filtered, left_on=['WTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'w_elo'})\ntourney_2017 = a.merge(elo_filtered, left_on=['LTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'l_elo'})\n#Usable output\nval_2017 = tourney_2017[tourney_2017['Season'] == 2017]\ntourney_2017 = tourney_2017[(tourney_2017['Season'] < 2017) & (tourney_2017['Season'] >= 2013)]\ntest_2017 = tourney_2017[tourney_2017['Season'] == 2016]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_2016 = tournament_result[(tournament_result['Season'] <= 2016) & (tournament_result['Season'] >= 2012)]\nelo_filtered = season_elos[(season_elos['Season'] <= 2016) & (season_elos['Season'] >= 2012)]\na = tourney_2016.merge(elo_filtered, left_on=['WTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'w_elo'})\ntourney_2016 = a.merge(elo_filtered, left_on=['LTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'l_elo'})\n#Usable output\nval_2016 = tourney_2016[tourney_2016['Season'] == 2016]\ntourney_2016 = tourney_2016[(tourney_2016['Season'] < 2016) & (tourney_2016['Season'] >= 2012)]\ntest_2016 = tourney_2016[tourney_2016['Season'] == 2015]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_2015 = tournament_result[(tournament_result['Season'] <= 2015) & (tournament_result['Season'] >= 2011)]\nelo_filtered = season_elos[(season_elos['Season'] <= 2015) & (season_elos['Season'] >= 2011)]\na = tourney_2015.merge(elo_filtered, left_on=['WTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'w_elo'})\ntourney_2015 = a.merge(elo_filtered, left_on=['LTeamID','Season'], right_on=['team_id', 'Season']).drop(['team_id'], axis=1).rename(columns={'season_elo':'l_elo'})\n#Usable output\nval_2015 = tourney_2015[tourney_2015['Season'] == 2015]\ntourney_2015 = tourney_2015[(tourney_2015['Season'] < 2015) & (tourney_2015['Season'] >= 2011)]\ntest_2015 = tourney_2015[tourney_2015['Season'] == 2014]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"voting_clf = VotingClassifier(estimators=[\n ('log_clf', LogisticRegression(penalty='l2', fit_intercept=False, C=0.0001,\n verbose=False, max_iter=1000, solver='lbfgs')),\n ('svm_clf',SVC(probability=True)),\n ('dt_clf',DecisionTreeClassifier(random_state=666))], voting='soft')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for reg, tourney, val in zip([reg_2015, reg_2016, reg_2017, reg_2018, reg_2019],[tourney_2015, tourney_2016, tourney_2017, tourney_2018, tourney_2019],[val_2015, val_2016, val_2017, val_2018, val_2019]):\n featureAddition(reg, elo=True)\n featureAddition(tourney, elo=True)\n featureAddition(val, elo=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calcRanking(reg_2015, last_day_ranking_2015)\ncalcRanking(tourney_2015, last_day_ranking_2015)\n\ncalcRanking(reg_2016, last_day_ranking_2016)\ncalcRanking(tourney_2016, last_day_ranking_2016)\n\ncalcRanking(reg_2017, last_day_ranking_2017)\ncalcRanking(tourney_2017, last_day_ranking_2017)\n\ncalcRanking(reg_2018, last_day_ranking_2018)\ncalcRanking(tourney_2018, last_day_ranking_2018)\n\ncalcRanking(reg_2019, last_day_ranking_2019)\ncalcRanking(tourney_2019, last_day_ranking_2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature = ['log_lower_elo','log_higher_elo']\n# feature = ['log_lower_elo','log_higher_elo','log_lower_ranking','log_higher_ranking']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature engineering\n\n**After carefully experimented and play-around, I found that it's better not to bring the ranking information in in order to have a better model performance.**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Binning\n**This feature engineering technique turns out to even drag the model performance down. So it's deprecated**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# reg_2015['higher_elo_band'] = pd.cut(reg_2015['higher_elo'],3)\n# reg_2015['higher_elo_band'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# interval_1 = pd.Interval(left=832.325,right=1262.732)\n# interval_2 = pd.Interval(left=1262.732,right=1691.853)\n# interval_3 = pd.Interval(left=1691.853,right=2120.973)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reg_2015.loc[(reg_2015['higher_elo_band'] == interval_1),'higher_elo'] = 1\n# reg_2015.loc[(reg_2015['higher_elo_band'] == interval_2),'higher_elo'] = 2\n# reg_2015.loc[(reg_2015['higher_elo_band'] == interval_3),'higher_elo'] = 3\n\n# reg_2015['higher_elo'] = reg_2015['higher_elo'].astype(int)\n# reg_2015.drop('higher_elo_band', axis=1, inplace=True)\n\n# reg_2015 = pd.get_dummies(reg_2015, columns=['higher_elo'], prefix='higher_elo')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Standardization**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Standardization on elo\nfor df in [reg_2015, reg_2016, reg_2017, reg_2018, reg_2019, val_2015, val_2016, val_2017, val_2018, val_2019]:\n df['standardized_lower_elo'] = (df['lower_elo'] - df['lower_elo'].mean()) / df['lower_elo'].std()\n df['standardized_higher_elo'] = (df['higher_elo'] - df['higher_elo'].mean()) / df['higher_elo'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #Standardization on ranking\n# for df in [reg_2015, reg_2016, reg_2017, reg_2018, reg_2019, val_2015, val_2016, val_2017, val_2018, val_2019]:\n# df['standardized_lower_ranking'] = (df['lower_ranking'] - df['lower_ranking'].mean()) / df['lower_ranking'].std()\n# df['standardized_higher_ranking'] = (df['higher_ranking'] - df['higher_ranking'].mean()) / df['higher_ranking'].std()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Log transform**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Log transform on elo\nfor df in [reg_2015, reg_2016, reg_2017, reg_2018, reg_2019, val_2015, val_2016, val_2017, val_2018, val_2019]:\n df['log_lower_elo'] = (df['standardized_lower_elo']-df['standardized_lower_elo'].min()+1) .transform(np.log)\n df['log_higher_elo'] = (df['standardized_higher_elo']-df['standardized_higher_elo'].min()+1) .transform(np.log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #Log transform on ranking\n# for df in [reg_2015, reg_2016, reg_2017, reg_2018, reg_2019, val_2015, val_2016, val_2017, val_2018, val_2019]:\n# df['log_lower_ranking'] = (df['standardized_lower_elo']-df['standardized_lower_elo'].min()+1) .transform(np.log)\n# df['log_higher_ranking'] = (df['standardized_higher_elo']-df['standardized_higher_elo'].min()+1) .transform(np.log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(9,3))\nreg_2015['standardized_higher_elo'].hist()\nplt.title('standardized_higher_elo')\nplt.show()\n# plt.figure(figsize=(9,3))\n# reg_2015['lower_ranking'].hist()\n# plt.figure(figsize=(9,3))\n# reg_2015['log_lower_ranking'].hist()\nplt.figure(figsize=(9,3))\nplt.title('standardized_higher_elo')\nreg_2015['higher_elo'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Here we barely see the difference between the distribution of standardized and original higher elo data.**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"

Submission

","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_seed = seedSeparation([2015,2016,2017,2018,2019])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import combinations\nfrom itertools import permutations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generateTourneySubFile(year):\n id_list = []\n comb = combinations(tourney_seed[year-2015]['TeamID'],2)\n for i in list(comb):\n firstTeam = i[0]\n secondTeam = i[1]\n if firstTeam > secondTeam:\n firstTeam = i[1]\n secondTeam = i[0]\n id_list.append(str(year) + '_' + str(firstTeam) + '_' + str(secondTeam))\n df = pd.DataFrame({'ID':id_list})\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_2015 = generateTourneySubFile(2015)\npred_2016 = generateTourneySubFile(2016)\npred_2017 = generateTourneySubFile(2017)\npred_2018 = generateTourneySubFile(2018)\npred_2019 = generateTourneySubFile(2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"elo_2015 = season_elos.loc[season_elos['Season'] == 2015]\nelo_2016 = season_elos.loc[season_elos['Season'] == 2016]\nelo_2017 = season_elos.loc[season_elos['Season'] == 2017]\nelo_2018 = season_elos.loc[season_elos['Season'] == 2018]\nelo_2019 = season_elos.loc[season_elos['Season'] == 2019]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def appendElo(df_pred, df_elo):\n for index, row in df_pred.iterrows():\n season, first, second = getTeam(df_pred, index)\n first_team_elo = round(float(df_elo.loc[df_elo['team_id'] == int(first)]['season_elo'].values),2)\n second_team_elo = round(float(df_elo.loc[df_elo['team_id'] == int(second)]['season_elo'].values),2)\n df_pred.loc[index, 'lower_elo'] = first_team_elo\n df_pred.loc[index, 'higher_elo'] = second_team_elo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for pred,elo, val,ranking in zip([pred_2015, pred_2016, pred_2017, pred_2018, pred_2019],[elo_2015, elo_2016, elo_2017, elo_2018, elo_2019],[val_2015,val_2016,val_2017,val_2018,val_2019],[last_day_ranking_2015, last_day_ranking_2016,last_day_ranking_2017,last_day_ranking_2018,last_day_ranking_2019]):\n appendElo(pred, elo)\n calcRanking(pred, ranking)\n calcRanking(val, ranking)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Standardization\nfor df in [pred_2015, pred_2016, pred_2017, pred_2018, pred_2019]:\n# df['standardized_lower_ranking'] = (df['lower_ranking'] - df['lower_ranking'].mean()) / df['lower_ranking'].std()\n# df['standardized_higher_ranking'] = (df['higher_ranking'] - df['higher_ranking'].mean()) / df['higher_ranking'].std()\n df['standardized_lower_elo'] = (df['lower_elo'] - df['lower_elo'].mean()) / df['lower_elo'].std()\n df['standardized_higher_elo'] = (df['higher_elo'] - df['higher_elo'].mean()) / df['higher_elo'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Log transform\nfor df in [pred_2015, pred_2016, pred_2017, pred_2018, pred_2019]:\n df['log_lower_elo'] = (df['standardized_lower_elo']-df['standardized_lower_elo'].min()+1) .transform(np.log)\n df['log_higher_elo'] = (df['standardized_higher_elo']-df['standardized_higher_elo'].min()+1) .transform(np.log)\n# df['log_lower_ranking'] = (df['standardized_lower_ranking']-df['standardized_lower_ranking'].min()+1) .transform(np.log)\n# df['log_higher_ranking'] = (df['standardized_higher_ranking']-df['standardized_higher_ranking'].min()+1) .transform(np.log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for df, year in zip([pred_2015, pred_2016, pred_2017, pred_2018, pred_2019],range(2015,2020)):\n# applyEff_season(df,year)\n# df['OEff_diff'] = df['Team1_OEff'] - df['Team2_OEff']\n# df['DEff_diff'] = df['Team1_DEff'] - df['Team2_DEff']\n# df.drop(['Team1_OEff','Team1_DEff','Team2_OEff','Team2_DEff'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# temp_2015 = pd.concat([reg_2015, test_2015],ignore_index=True)\n# print('Feature addition....')\n# featureAddition(val_2015)\n# print('Done!')\nX_train = reg_2015[feature]\ny_train = reg_2015[['lower_win']]\nprint('Fitting....')\nvoting_clf.fit(X_train, y_train)\nprint('Done!')\nappendPred(pred_2015)\n# pred_2015.drop(['lower_elo','higher_elo','lower_ranking','higher_ranking'],axis=1, inplace=True)\nprint(log_loss(val_2015['lower_win'], voting_clf.predict_proba(val_2015[feature])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# featureAddition(test_2016, elo=True)\n# temp_2016 = pd.concat([reg_2016, test_2016],ignore_index=True)\nX_train = reg_2016[feature]\ny_train = reg_2016[['lower_win']]\nvoting_clf.fit(X_train, y_train)\nappendPred(pred_2016)\n# pred_2016.drop(feature,axis=1, inplace=True)\nprint(log_loss(val_2016['lower_win'], voting_clf.predict_proba(val_2016[feature])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# featureAddition(test_2017, elo=True)\n# temp_2017 = pd.concat([reg_2017, test_2017],ignore_index=True)\nX_train = reg_2017[feature]\ny_train = reg_2017[['lower_win']]\nvoting_clf.fit(X_train, y_train)\nappendPred(pred_2017)\n# pred_2017.drop(feature,axis=1, inplace=True)\nprint(log_loss(val_2017['lower_win'], voting_clf.predict_proba(val_2017[feature])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# featureAddition(test_2018, elo=True)\n# temp_2018 = pd.concat([reg_2018, test_2018],ignore_index=True)\nX_train = reg_2018[feature]\ny_train = reg_2018[['lower_win']]\nvoting_clf.fit(X_train, y_train)\nappendPred(pred_2018)\n# pred_2018.drop(feature,axis=1, inplace=True)\nprint(log_loss(val_2018['lower_win'], voting_clf.predict_proba(val_2018[feature])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# featureAddition(test_2019, elo=True)\n# temp_2019 = pd.concat([reg_2019, test_2019],ignore_index=True)\nX_train = reg_2019[feature]\ny_train = reg_2019[['lower_win']]\nvoting_clf.fit(X_train, y_train)\nappendPred(pred_2019)\n# pred_2019.drop(feature,axis=1, inplace=True)\nprint(log_loss(val_2019['lower_win'], voting_clf.predict_proba(val_2019[feature])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_file = pd.concat([pred_2015, pred_2016, pred_2017, pred_2018, pred_2019])\nfor index, row in submission_file.iterrows():\n if row['Pred'] > 0.95:\n submission_file.loc[index,'Pred'] = 0.95\n if row['Pred'] < 0.05:\n submission_file.loc[index,'Pred'] = 0.05","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission_file = submission_file[['ID','Pred']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission_file.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## P.S. Manually guessing technique\n\n* Get team name\n* Get team last day ranking\n* Get team stats\n* Get team pace(by calculating poss first)","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"gameReport('2019_1192_1293')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get team Name\ndef gameReport(ID):\n field = {'Team Name','Last Day Ranking','Avg Score','AScore allowed','Avg OR','Avg DR'}\n comparison = pd.DataFrame(columns={'Lower team', 'Higher team'}, index=field)\n season = int(ID[0:4])\n first = int(ID[5:9])\n second = int(ID[10:14])\n lower_team_name = teams[teams['TeamID'] == first]['TeamName'].values[0]\n higher_team_name = teams[teams['TeamID'] == second]['TeamName'].values[0]\n comparison.loc['Team Name'] = [lower_team_name, higher_team_name]\n \n #Last day ranking\n lower_rank = last_day_ranking_2019[(last_day_ranking_2019['TeamID'] == first) & (last_day_ranking_2019['Season'] == season)]['Avg_rank'].values[0]\n higher_rank = last_day_ranking_2019[(last_day_ranking_2019['TeamID'] == second) & (last_day_ranking_2019['Season'] == season)]['Avg_rank'].values[0]\n comparison.loc['Last Day Ranking'] = [lower_rank, higher_rank]\n \n #Avg game stat\n l_score, l_off_reb, l_def_reb, l_to, l_asst, l_stl, l_blk, l_fga3, l_pct3= gameSum(season, first)\n h_score, h_off_reb, h_def_reb, h_to, h_asst, h_stl, h_blk, h_fga3, h_pct3= gameSum(season, second)\n comparison.loc['Avg Score'] = [l_score,h_score]\n comparison.loc['Avg OR'] = [l_off_reb, h_off_reb]\n comparison.loc['Avg DR'] = [l_def_reb, h_off_reb]\n comparison.loc['Avg TO'] = [l_to, h_to]\n comparison.loc['Avg Asst'] = [l_asst, h_asst]\n comparison.loc['Avg Stl'] = [l_stl, h_stl]\n comparison.loc['Avg Blk'] = [l_blk, h_blk]\n comparison.loc['Avg FGA3'] = [l_fga3, h_fga3]\n comparison.loc['3PT PCT'] = [l_pct3, h_pct3]\n \n \n #Ranking trend\n# plotAcrossSeasons(season, first)\n# plotAcrossSeasons(season, second)\n \n return comparison","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}