{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport glob\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom scipy.optimize import minimize\nimport itertools","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Input data\n\nThanks author of [this notebook](https://www.kaggle.com/artgor/march-madness-2020-ncaam-eda-and-baseline) for idea"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict = {}\nfor i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/*'):\n    name = i.split('/')[-1].split('.')[0]\n    if name != 'MTeamSpellings':\n        data_dict[name] = pd.read_csv(i)\n    else:\n        data_dict[name] = pd.read_csv(i, encoding='cp1252')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_seed_num(df_input, df_seeds=data_dict['MNCAATourneySeeds']):\n    \n    df = df_input.copy()\n\n    df = df.join(df_seeds.set_index(['Season', 'TeamID'])['Seed'].rename('Wseed'), on=['Season', 'WTeamID'])\n    df = df.join(df_seeds.set_index(['Season', 'TeamID'])['Seed'].rename('Lseed'), on=['Season', 'LTeamID'])\n    df['WRegionSeed'] = df['Wseed'].apply(lambda x: int(x[1:3]))\n    df['LRegionSeed'] = df['Lseed'].apply(lambda x: int(x[1:3]))\n\n    return df\n\n\ndef add_game_round(Seed_DayNum, df_rounds=data_dict['MNCAATourneySeedRoundSlots']):\n    \n    Seed = Seed_DayNum[:-3]\n    DayNum = int(Seed_DayNum[-3:])\n    GameRound = list(df_rounds[(df_rounds['Seed'] == Seed) & (df_rounds['EarlyDayNum'] <= DayNum) & (df_rounds['LateDayNum'] >= DayNum)]['GameRound'].unique()\n                    )\n    if len(GameRound) > 1:\n        print('error')\n        \n    return str(GameRound[0])\n\n\ndef winrate_by_seed(df_input, start_year=0, end_year=0, window=5):\n    \n    df = df_input.copy()\n    df = add_seed_num(df)\n    df['SameSeed'] = (df['WRegionSeed'] == df['LRegionSeed']).astype(int)\n    df = df[df['SameSeed'] == 0].copy() \n    df_win = df[['Season', 'WRegionSeed']].copy().rename(columns={'WRegionSeed': 'Seed'})\n    df_win['Winrate'] = 1\n    df_lose = df[['Season', 'LRegionSeed']].copy().rename(columns={'LRegionSeed': 'Seed'})\n    df_lose['Winrate'] = 0\n    df_concat = pd.concat((df_win, df_lose))\n    years_to_predict = np.arange(start_year, end_year+1)\n    result = pd.DataFrame()\n    \n    for year in years_to_predict:\n        \n        df_window = df_concat[(df_concat['Season'] < year) & (df_concat['Season'] >= year-window)].copy()\n        df_window['Season'] = year\n        result = pd.concat((result, df_window))\n    \n    return result.groupby(['Season', 'Seed'], as_index=False)['Winrate'].mean()\n\n\ndef winrate_by_seed_linear(df_input, start_year=0, end_year=0, window=5):\n    \n    df = df_input.copy()\n    df = add_seed_num(df)\n    df = order_teams(df)\n    \n    years_to_predict = np.arange(start_year, end_year+1)\n    result = pd.DataFrame()\n    \n    for year in years_to_predict:\n        \n        df_window = df[(df['Season'] < year) & (df['Season'] >= year-window)].copy()\n        seeds = df_window['Team1Seed'].values\n        results = df_window['Result'].values\n        SE = lambda x: np.sum((x[0] * seeds + x[1] - results) ** 2)\n        coef = minimize(SE, (-0.05, 1), method='Nelder-Mead', tol=1e-6)['x']\n        year_results = pd.DataFrame()\n        year_results['Seed'] = np.arange(1,17)\n        year_results['Winrate'] = year_results['Seed'].apply(lambda x: x * coef[0] + coef[1])\n        year_results['Season'] = year   \n        result = pd.concat((result, year_results))\n    \n    return result\n\n\ndef order_teams(df_input):\n    \n    df = df_input.copy()\n    df['Team1'] = np.where(df.WTeamID < df.LTeamID, df.WTeamID, df.LTeamID)\n    df['Team2'] = np.where(df.WTeamID > df.LTeamID, df.WTeamID, df.LTeamID)\n    df['Team1Seed'] = np.where(df.WTeamID < df.LTeamID, df.WRegionSeed, df.LRegionSeed)\n    df['Team2Seed'] = np.where(df.WTeamID > df.LTeamID, df.WRegionSeed, df.LRegionSeed)\n    df['Result'] = np.where(df.WTeamID < df.LTeamID, 1, 0)\n    \n    return df\n\n\ndef prepare_train_dataset(start_year=2010, end_year=2014, approximation=None):\n    train = data_dict['MNCAATourneyCompactResults'][(data_dict['MNCAATourneyCompactResults']['Season'] <= end_year) \\\n                                                    & (data_dict['MNCAATourneyCompactResults']['Season'] >= start_year)].copy()\n    \n    train = add_seed_num(train)\n    train = order_teams(train)\n    \n    if approximation == 'linear':\n        winrate_by_seed_train = winrate_by_seed_linear(data_dict['MNCAATourneyCompactResults'], start_year, end_year)\n        \n    else: \n        winrate_by_seed_train = winrate_by_seed(data_dict['MNCAATourneyCompactResults'], start_year, end_year)\n        \n    train = train.join(winrate_by_seed_train.set_index(['Season', 'Seed'])['Winrate'].rename('Team1Winrate'), on=['Season', 'Team1Seed'])\n    train = train.join(winrate_by_seed_train.set_index(['Season', 'Seed'])['Winrate'].rename('Team2Winrate'), on=['Season', 'Team2Seed'])    \n    train['WinrateDiff'] = train['Team1Winrate'] - train['Team2Winrate']\n    train_X = train[['Team1Winrate', 'Team2Winrate']]\n    train_y = train['Result']\n    \n    return train_X, train_y\n\n\ndef prepare_test_dataset(df_input, start_year=2015, end_year=2019):\n    \n    df = df_input.copy()\n    \n    df['Season'] = df['ID'].apply(lambda x: int(x.split('_')[0]))\n    df['Team1'] = df['ID'].apply(lambda x: int(x.split('_')[1]))\n    df['Team2'] = df['ID'].apply(lambda x: int(x.split('_')[2]))\n    \n    df_seeds=data_dict['MNCAATourneySeeds']\n    \n    df = df.join(df_seeds.set_index(['Season', 'TeamID'])['Seed'].rename('Team1seed'), on=['Season', 'Team1'])\n    df = df.join(df_seeds.set_index(['Season', 'TeamID'])['Seed'].rename('Team2seed'), on=['Season', 'Team2'])\n    df['Team1RegionSeed'] = df['Team1seed'].apply(lambda x: int(x[1:3]))\n    df['Team2RegionSeed'] = df['Team2seed'].apply(lambda x: int(x[1:3]))\n    \n    winrate_by_seed = winrate_by_seed_linear(data_dict['MNCAATourneyCompactResults'], start_year, end_year)\n\n    \n    df = df.join(winrate_by_seed.set_index(['Season', 'Seed'])['Winrate'].rename('Team1Winrate'), on=['Season', 'Team1RegionSeed'])\n    df = df.join(winrate_by_seed.set_index(['Season', 'Seed'])['Winrate'].rename('Team2Winrate'), on=['Season', 'Team2RegionSeed'])   \n    \n    return df['ID'], df[['Team1Winrate', 'Team2Winrate']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_TourneyResultsSeeds = add_seed_num(data_dict['MNCAATourneyCompactResults'])\ndf_TourneyResultsSeeds['GameRound'] = (df_TourneyResultsSeeds.Wseed + df_TourneyResultsSeeds.DayNum.astype(str)).apply(lambda x : add_game_round(x)).astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Short analysis of seed performance\n\nThe higher seed will win at roughly 75% of the time in the first round, while that number will fall to around 70% for both the second roundand Sweet 16. The winning percentage of the higher seed then falls off considerably during the Elite 8 to below 50%.\n\nThe Final Four and Championship game have a higher proportion of games where the same seeds play. Still, the lower ranked team will only win around 20% of the time in the final four, and only approximately 5-10% of championship games.\n\nCheck [this notebook](https://www.kaggle.com/jaseziv83/moreyball-in-the-college-game-a-full-ncaa-eda)\n\nLet's make a visualisation of results of games grouped by seeds and rounds"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, ncols=3, figsize=(30, 20))\nSeeds_groupby = df_TourneyResultsSeeds.groupby(['WRegionSeed', 'LRegionSeed', 'GameRound'], as_index=False)['DayNum'].count()\n\nfor GameRound in range(1, 7):\n    axes = ax[(GameRound-1) // 3, (GameRound-1) % 3]\n    sns.heatmap(Seeds_groupby[Seeds_groupby['GameRound'] == GameRound].pivot(index='WRegionSeed', columns='LRegionSeed', values='DayNum') \\\n                .fillna(0).astype(int), annot=True, cmap='Blues', fmt='2g', ax=axes)\n    axes.set_title('Round ' + str(GameRound))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As mentioned above we can make good predictions for first 3 rounds, because higher seed usually wins. See first round seed 1 vs seed 16 (139 wins and 1 lose for seed 1). \n\nAnother thing that we should pay attention is that higher seed can have worse results than it's neighbour see first round seed 8 (68 wins) against seed 9 (72 wins). We don't expect this and we deal with this issue later."},{"metadata":{},"cell_type":"markdown","source":"## Training model"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Baseline:', -np.log(0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, train_y = prepare_train_dataset(start_year=2010, end_year=2014)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf1 = LogisticRegression()\ncv1 = cross_val_score(clf1, train_X.values, train_y.values, cv=5)\n\ncv1.mean(), cv1.std()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Great! Our model better than constant prediction :)"},{"metadata":{},"cell_type":"markdown","source":"## Improve model"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_TourneyResultsSeeds = order_teams(df_TourneyResultsSeeds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In our model we used winnig percentage of each seed against the others.\n\nLet's draw this in one plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.arange(1,17), df_TourneyResultsSeeds.groupby('Team1Seed')['Result'].mean().values)\nplt.ylabel('Winrate')\nplt.xlabel('Seed')\nplt.title('Winrate by seed');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"WOW it's very close to linear relationship, but something strange happens with 11 and 12 seeds. They play much better than I expected. If you know what is the reason of it please comment.\n\nAnyway let's make linear approximation of this function. And try to use these approximated winning percentage in our model."},{"metadata":{"trusted":true},"cell_type":"code","source":"seeds = df_TourneyResultsSeeds['Team1Seed'].values\nresults = df_TourneyResultsSeeds['Result'].values\nSE = lambda x: np.sum((x[0] * seeds + x[1] - results) ** 2)\ncoef = minimize(SE, (-0.05, 1), method='Nelder-Mead', tol=1e-6)['x']\napproximation = np.arange(1,17) * coef[0] + coef[1]\nplt.plot(np.arange(1,17), df_TourneyResultsSeeds.groupby('Team1Seed')['Result'].mean().values, label='True')\nplt.plot(np.arange(1,17), approximation, label='Linear approximation')\nplt.legend()\nplt.ylabel('Winrate')\nplt.xlabel('Seed')\nplt.title('Winrate by seed');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train another model"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X_linear, train_y = prepare_train_dataset(start_year=2010, end_year=2014, approximation='linear')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf2 = LogisticRegression()\ncv2 = cross_val_score(clf2, train_X_linear.values, train_y.values, cv=5)\n\ncv2.mean(), cv2.std()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Awesome! This model better than previous one. Let's make visualisation of results."},{"metadata":{"trusted":true},"cell_type":"code","source":"x_s = np.array(list(itertools.product(np.linspace(0, 0.9, 100), np.linspace(0, 0.9, 100))))\n\nclf1.fit(train_X, train_y)\ny_s = clf1.predict(train_X)\ny_s_proba = clf1.predict_proba(x_s)[:,1]\n\nclf2.fit(train_X_linear, train_y)\ny_s_linear = clf2.predict(train_X_linear)\ny_s_proba_linear = clf2.predict_proba(x_s)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(ncols=2, figsize=(16,4))\n\nax[0].scatter(x_s[:, 0], x_s[:, 1], c=y_s_proba, cmap='bwr')\nax[0].scatter(train_X.values[:, 0], train_X.values[:, 1], alpha=0.3, s=100, c=(train_y == y_s), cmap='bone')\nax[0].set_title('True winrate')\nax[0].set_xlabel('First team winrate')\nax[0].set_ylabel('Second team winrate')\n\nax[1].scatter(x_s[:, 0], x_s[:, 1], c=y_s_proba_linear, cmap='bwr')\nax[1].scatter(train_X_linear.values[:, 0], train_X_linear.values[:, 1], alpha=0.3, s=100, c=(train_y == y_s_linear), cmap='bone')\nax[1].set_title('Winrate linear approximation')\nax[1].set_xlabel('First team winrate')\nax[1].set_ylabel('Second team winrate');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the right plot we can differ first round (diagonal from (0, 0.8) to (0.8, 0)) from other games"},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices, dt_test = prepare_test_dataset(df_submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = LogisticRegression()\nclf.fit(train_X_linear, train_y)\npredictions = clf.predict_proba(dt_test)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['Pred'] = predictions\ndf_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thanks for reading this! Your comments are appreciated."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}