{"cells": [{"cell_type": "code", "execution_count": 1, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["# Auto-generated, do not edit!\n", "# One final run.\n", "INIT = [('chariots17/using-xgboost-lgb-to-predict',\n", " 'using-xgboost-lgb-to-predict',\n", " 8009069,\n", " ' Using XGBOOST&lgb to predict',\n", " '2020-02-19T10:19:05.523Z',\n", " '\u8c01\u4e0d\u91cd\u8981',\n", " 'chariots17',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/2519363-kg.jpg',\n", " 0.26295,\n", " 17,\n", " '\ud83e\udd49'),\n", " ('hiromoon166/2020-women-s-starter-kernel',\n", " '2020-women-s-starter-kernel',\n", " 7980891,\n", " \"2020 Women's Starter Kernel\",\n", " '2020-02-15T09:33:14.507Z',\n", " 'hiromu',\n", " 'hiromoon166',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1497263-kg.jpg',\n", " 0.44527,\n", " 17,\n", " '\ud83e\udd49'),\n", " ('code1110/ncaaw20-eda-and-nn-lgb-catb-starter',\n", " 'ncaaw20-eda-and-nn-lgb-catb-starter',\n", " 8016864,\n", " '[NCAAW20] EDA and NN+LGB+CatB starter',\n", " '2020-02-21T12:20:13.09Z',\n", " 'katsu1110',\n", " 'code1110',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/590240-fb.jpg',\n", " 0.19793,\n", " 15,\n", " '\ud83e\udd49'),\n", " ('code1110/ncaaw20-finally-no-leak-starter-with-lgb',\n", " 'ncaaw20-finally-no-leak-starter-with-lgb',\n", " 8084358,\n", " '[NCAAW20] (finally) no-leak starter with LGB',\n", " '2020-02-24T00:37:52.613Z',\n", " 'katsu1110',\n", " 'code1110',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/590240-fb.jpg',\n", " 0.48157,\n", " 11,\n", " '\ud83e\udd49'),\n", " ('moradnejad/eda-plus-nn-lgb-best-score-with-no-leak',\n", " 'eda-plus-nn-lgb-best-score-with-no-leak',\n", " 8279350,\n", " 'EDA plus NN+LGB: BEST score with NO leak',\n", " '2020-03-06T16:24:45.533Z',\n", " 'Moradnejad',\n", " 'moradnejad',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/2129486-kg.png',\n", " 0.1822,\n", " 9,\n", " '\ud83e\udd49'),\n", " ('takaishikawa/no-ml-modeling-ncaaw2020',\n", " 'no-ml-modeling-ncaaw2020',\n", " 8087017,\n", " 'No ML Modeling - NCAAW2020',\n", " '2020-02-23T11:33:06.42Z',\n", " 'Tak',\n", " 'takaishikawa',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1015481-kg.jpg',\n", " 0.45292,\n", " 8,\n", " '\ud83e\udd49'),\n", " ('a45632/2020-starter-kernel-women-improved',\n", " '2020-starter-kernel-women-improved',\n", " 7981323,\n", " '2020 Starter Kernel Women Improved',\n", " '2020-02-15T11:06:30.223Z',\n", " 'Eric Vos',\n", " 'a45632',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/833052-kg.jpg',\n", " 0.40199,\n", " 8,\n", " '\ud83e\udd49'),\n", " ('immvab/nn-starter-tensorflow',\n", " 'nn-starter-tensorflow',\n", " 8012595,\n", " 'NN Starter Tensorflow',\n", " '2020-02-17T18:52:03.137Z',\n", " 'Vaibhav Birla',\n", " 'immvab',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/4422823-kg.png',\n", " 0.43906,\n", " 6,\n", " ' '),\n", " ('vbmokin/mm-ncaaw-lgb-xgb-regr',\n", " 'mm-ncaaw-lgb-xgb-regr',\n", " 8310908,\n", " 'MM NCAAW - LGB, XGB & Regr',\n", " '2020-03-13T13:14:50.523Z',\n", " 'Vitalii Mokin',\n", " 'vbmokin',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1798416-gp.jpg',\n", " 0.12234,\n", " 4,\n", " ' '),\n", " ('darwinwin/ncaaw20-eda-and-nn-lgb-catb-starter-7c65f8',\n", " 'ncaaw20-eda-and-nn-lgb-catb-starter-7c65f8',\n", " 8192805,\n", " '[NCAAW20] EDA and NN+LGB+CatB starter 7c65f8',\n", " '2020-02-29T14:11:26.193Z',\n", " 'darwinwin',\n", " 'darwinwin',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/4538297-kg.jpg',\n", " 0.20489,\n", " 4,\n", " ' '),\n", " ('scirpus/last-year-plus-mens-gp',\n", " 'last-year-plus-mens-gp',\n", " 8178582,\n", " 'Last Year plus mens GP',\n", " '2020-02-28T12:54:10.947Z',\n", " 'Scirpus',\n", " 'scirpus',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/241438-fb.jpg',\n", " 0.39887,\n", " 4,\n", " ' '),\n", " ('hamditarek/ncaaw20-eda-and-nn-lgb-catb-starter-7c65f8',\n", " 'ncaaw20-eda-and-nn-lgb-catb-starter-7c65f8',\n", " 8079138,\n", " '[NCAAW20] EDA and NN+LGB+CatB starter 7c65f8',\n", " '2020-02-21T18:29:06.693Z',\n", " 'Tarek Hamdi',\n", " 'hamditarek',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1673888-kg.jpg',\n", " 0.20404,\n", " 4,\n", " ' '),\n", " ('darwinwin/ncaaw20-eda-and-nn-lgb-catb-starter',\n", " 'ncaaw20-eda-and-nn-lgb-catb-starter',\n", " 8191645,\n", " '[NCAAW20] EDA and NN+LGB+CatB starter',\n", " '2020-02-29T13:19:27.103Z',\n", " 'darwinwin',\n", " 'darwinwin',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/4538297-kg.jpg',\n", " 0.22206,\n", " 3,\n", " ' '),\n", " ('sakana/simple-feature-enginering-with-lightgbm',\n", " 'simple-feature-enginering-with-lightgbm',\n", " 8045032,\n", " 'Simple Feature Enginering with LightGBM',\n", " '2020-02-26T11:58:20.973Z',\n", " 'sakana',\n", " 'sakana',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1377128-kg.jpg',\n", " 0.39544,\n", " 3,\n", " ' '),\n", " ('chariots17/artificial-nerual-network-keras',\n", " 'artificial-nerual-network-keras',\n", " 7995656,\n", " 'Artificial Nerual Network(keras)',\n", " '2020-02-17T11:01:42.583Z',\n", " '\u8c01\u4e0d\u91cd\u8981',\n", " 'chariots17',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/2519363-kg.jpg',\n", " 0.44187,\n", " 3,\n", " ' '),\n", " ('miklgr500/keras-nn-ncaaw',\n", " 'keras-nn-ncaaw',\n", " 8332826,\n", " 'Keras NN NCAAW',\n", " '2020-03-10T09:12:53.043Z',\n", " 'Michael Kazachok',\n", " 'miklgr500',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1210896-kg.jpg',\n", " 0.3608,\n", " 2,\n", " ' '),\n", " ('miklgr500/bayesian-neural-network-in-keras-ncaaw',\n", " 'bayesian-neural-network-in-keras-ncaaw',\n", " 8303877,\n", " 'Bayesian Neural Network in Keras NCAAW',\n", " '2020-03-09T11:22:50.843Z',\n", " 'Michael Kazachok',\n", " 'miklgr500',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/1210896-kg.jpg',\n", " 0.42929,\n", " 2,\n", " ' '),\n", " ('tnmasui/ncaaw-2020-lgb-w-fe-on-three-datasets',\n", " 'ncaaw-2020-lgb-w-fe-on-three-datasets',\n", " 8212330,\n", " 'NCAAW 2020 - LGB w/ FE on three Datasets',\n", " '2020-03-02T04:59:06.727Z',\n", " 'Tomonori Masui',\n", " 'tnmasui',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/427399-kg.JPG',\n", " 0.44515,\n", " 2,\n", " ' '),\n", " ('christoffer/simple-k-optimized-elo-model-ncaaw',\n", " 'simple-k-optimized-elo-model-ncaaw',\n", " 8167258,\n", " 'Simple k-optimized Elo model (NCAAW)',\n", " '2020-02-27T16:12:19.817Z',\n", " 'Christoffer Karlsson',\n", " 'christoffer',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/94532-kg.png',\n", " 0.53183,\n", " 2,\n", " ' '),\n", " ('johnt666/no-leaks-what-do-you-think',\n", " 'no-leaks-what-do-you-think',\n", " 8094832,\n", " 'No leaks, What do you think?',\n", " '2020-03-01T01:04:45.427Z',\n", " 'Taco',\n", " 'johnt666',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/2015375-kg.jpg',\n", " 0.61052,\n", " 1,\n", " ' '),\n", " ('mrboupp/first-submission-using-lightgbm',\n", " 'first-submission-using-lightgbm',\n", " 8315903,\n", " 'First Submission using Lightgbm',\n", " '2020-03-09T14:36:46.76Z',\n", " 'marbou',\n", " 'mrboupp',\n", " 'https://storage.googleapis.com/kaggle-avatars/thumbnails/3563243-kg.jpg',\n", " 0.28961,\n", " 0,\n", " ' ')]\n"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Blend NCAAW with 2020 Vision\n", "\n", "Yes, this Notebook will blend all the public stage 1 solutions!\n", "\n", "Along the way, I will plot all their predictions in a 2D histogram (a heatmap), one for each year. (Which is, I hope you realise, the real purpose of this Notebook!) There are links to the actual Notebooks — so open them up — this gives you another reason to upvote them: marks for aesthetics :)\n", "\n", "For stage 1 only 63 values per year are used as the ground truth, making it very easy to overfit. What is lurking behind these leaderboard scores? I'm not going to comment on individual entries, but it is easy to see when submissions have a high *noise* level, deviating wildly from the team seeds. Submissions used are from the last run of each source Notebook and some will be *work in progress*.\n", "\n", "See [here](https://www.kaggle.com/jtrotman/meta-march-mania) for a summary of the history of the March Madness Kaggle competitions, including histograms of actual scores after stage 2.\n", "\n", "## Plot Format\n", "\n", "The plot format is introduced [here](https://www.kaggle.com/c/march-machine-learning-mania-2017/discussion/30333).\n", "\n", "To recap: it is easiest to read the row for each team, where white means 50:50, red indicates probably winning, blue means probably losing, the deeper the color, the stronger the probability.\n", "\n", "In this Notebook the play-in matches are removed, leaving a 64x64 grid showing the four 16x16 regions, and all possible tournament matches. Note we only supply half of the grid in submissions: the other half is $1 - Pred$.\n", "\n", "\n", "## More Submission Heatmaps\n", "\n", "In previous years, Kaggle released **all** the stage 2 submission files before the action started: here they all are, **over 2,000 plots**, rendered in the same heatmap format :D\n", "\n", " - [2015](https://www.kaggle.com/jtrotman/beautiful-mania-2015)\n", " - [2016](https://www.kaggle.com/jtrotman/beautiful-mania-2016)\n", " - [2017](https://www.kaggle.com/jtrotman/beautiful-mania-2017)\n"]}, {"cell_type": "code", "execution_count": 2, "metadata": {"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_kg_hide-input": true, "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}, "outputs": [], "source": ["%matplotlib inline\n", "import pandas as pd\n", "import numpy as np\n", "import os, re, sys\n", "import matplotlib.pyplot as plt\n", "from IPython.core.display import HTML, Image\n", "\n", "TAG = 'NCAAW_2020'\n", "DIR = '../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1'\n", "PNG_DIR = '/kaggle/plots'\n", "OUTPUT_ZIP = f'{TAG}_stage1_plots.zip'\n", "\n", "os.makedirs(PNG_DIR, exist_ok=True)\n", "\n", "def read_results():\n", " res = pd.read_csv(f'{DIR}/WNCAATourneyCompactResults.csv').query('DayNum>=136')\n", " TC = ['WTeamID', 'LTeamID']\n", " res['Low'] = res[TC].min(1)\n", " res['High'] = res[TC].max(1)\n", " res['Truth'] = res.eval('Low==WTeamID').astype(int)\n", " res['Key'] = res.apply(lambda r: f'{r.Season}_{r.Low}_{r.High}', 1)\n", " res = res.set_index('Key')\n", " return res\n", "\n", "TEAMS = pd.read_csv(f'{DIR}/WTeams.csv', index_col=0)\n", "SEEDS = pd.read_csv(f'{DIR}/WNCAATourneySeeds.csv', index_col=2)\n", "RESULTS = read_results()\n", "\n", "TEAMS.shape, SEEDS.shape, RESULTS.shape"]}, {"cell_type": "code", "execution_count": 3, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["plt.rc('figure', figsize=(14, 14))\n", "plt.rc('font', size=12)\n", "\n", "ICOLS = ['i1', 'i2']\n", "\n", "\n", "def expand_sub(df):\n", " parts = df['id'].str.split('_')\n", " df['year'] = parts.str[0].astype(int)\n", " df['t1'] = parts.str[1].astype(int)\n", " df['t2'] = parts.str[2].astype(int)\n", " return df[['id', 'year', 't1', 't2', 'pred']].set_index('id')\n", "\n", "\n", "# return a submission in a standard form\n", "def read_sub(name):\n", " df = pd.read_csv(name)\n", " df.columns = df.columns.str.lower()\n", " return expand_sub(df)\n", "\n", "\n", "def log_loss(df):\n", " p = np.where(df.Truth, df.pred, 1 - df.pred)\n", " # clip low predictions to avoid infinite loss\n", " p = p.clip(min=1e-15)\n", " return (-np.log(p)).mean()\n", "\n", "\n", "def score_sub(sub):\n", " df = sub.join(RESULTS, how='inner')\n", " return log_loss(df)\n", "\n", "\n", "class Year:\n", "\n", " def __init__(self, res, seeds):\n", " self.res = res\n", " ids = set(res.Low) | set(res.High)\n", " self.nteams = len(ids)\n", " self.seeds = seeds[seeds.index.isin(ids)] # 64 teams\n", " self.seeds = self.seeds.join(TEAMS)\n", " self.inds = dict(zip(self.seeds.index, range(self.nteams)))\n", " # labels for each axis\n", " self.seeds['lx'] = self.seeds.Seed + \" \" + self.seeds.TeamName\n", " self.seeds['ly'] = self.seeds.TeamName + \" \" + self.seeds.Seed\n", " \n", " def add_inds(self, df):\n", " df = df.assign(i1=df.t1.map(self.inds), i2=df.t2.map(self.inds))\n", " df = df.dropna()\n", " df[ICOLS] = df[ICOLS].astype(int)\n", " return df\n", "\n", " def to_matrix(self, sub):\n", " sub = self.add_inds(sub)\n", " nteams = self.nteams\n", " m = np.ones((nteams, nteams)) * 0.5\n", " m[sub.i1, sub.i2] = sub.pred\n", " m[sub.i2, sub.i1] = 1 - sub.pred\n", " return m\n", "\n", " def heatmap(self, sub, filename, cmap=plt.cm.seismic):\n", " probs = self.to_matrix(sub)\n", " fig, ax = plt.subplots()\n", " heatmap = ax.pcolormesh(probs, vmin=0., vmax=1., cmap=cmap)\n", "\n", " ax.spines['top'].set_visible(False)\n", " ax.spines['right'].set_visible(False)\n", " ax.spines['bottom'].set_visible(False)\n", " ax.spines['left'].set_visible(False)\n", "\n", " ax.invert_yaxis()\n", " ax.tick_params(direction='out')\n", " ax.xaxis.tick_top()\n", " ax.yaxis.tick_left()\n", " plt.xticks(rotation=90)\n", "\n", " # put the major ticks at the middle of each cell\n", " ax.set_xticks(np.arange(self.nteams)+0.5, minor=False)\n", " ax.set_yticks(np.arange(self.nteams)+0.5, minor=False)\n", " ax.set_xticklabels(self.seeds['lx'])\n", " ax.set_yticklabels(self.seeds['ly'])\n", " plt.savefig(filename, bbox_inches='tight') \n", "\n", "\n", "YLIST = SEEDS.Season.unique()\n", "YEARS = { year: Year(RESULTS.query(f'Season=={year}'), SEEDS.query(f'Season=={year}')) for year in YLIST }"]}, {"cell_type": "code", "execution_count": 4, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["def reader(lst):\n", " for scriptUrl, currentUrlSlug, *rest in lst:\n", " bd = f'../input/{currentUrlSlug}'\n", " subs = {}\n", " try:\n", " files = os.listdir(bd)\n", " for f in files:\n", " if f.lower().endswith('.csv'):\n", " try:\n", " sub = read_sub(f'{bd}/{f}')\n", " subs[f] = sub\n", " except:\n", " pass\n", " except:\n", " pass\n", " if len(subs) > 0:\n", " yield [scriptUrl, currentUrlSlug] + rest + [subs]\n", "\n", "# Ensemble average\n", "sums = 0\n", "count = 0\n", "\n", "def show_years(sub, tag):\n", " display(HTML(\n", " f'

Prediction Stats

'\n", " f'

Log Loss: {score_sub(sub):.6f}'\n", " ))\n", " gb = sub.groupby('year')\n", " display(gb.pred.agg(['count', 'min', 'max']))\n", " for year, subdf in gb:\n", " display(HTML(\n", " f'

{year}

'\n", " f'

Log Loss: {score_sub(subdf):.6f}'\n", " ))\n", " YEARS[year].heatmap(subdf, f'{PNG_DIR}/{tag}_{year}')\n", " plt.show()\n", "\n", "for scriptUrl, currentUrlSlug, _id, title, lastRunTime, displayName, userName, thumbnailUrl, bestPublicScore, totalVotes, medal, subs in reader(INIT):\n", " display(HTML(\n", " f'

{title}

'\n", " f''\n", " f'

{displayName}'\n", " f'

{scriptUrl}'\n", " f'
Last run: {pd.to_datetime(lastRunTime).strftime(\"%c\")} '\n", " f'
Votes: {totalVotes} {medal}'\n", " f'
Best public log-loss: {bestPublicScore} '\n", " f'

'\n", " ))\n", " \n", " for csv, sub in subs.items():\n", " show_years(sub, f'{userName}_{currentUrlSlug}')\n", "\n", " sums += sub[['pred']].clip(0, 1)\n", " count += 1"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Ensemble\n", "\n", "Compute the mean average of all the submissions we managed to load."]}, {"cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": ["ensemble = (sums / count)\n", "count, score_sub(ensemble)"]}, {"cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": ["show_years(expand_sub(ensemble.reset_index()), f'{TAG}_ensemble')"]}, {"cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": ["ensemble.to_csv(f'{TAG}_blend.csv')"]}, {"cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": ["!7z a -bd -mmt4 {OUTPUT_ZIP} {PNG_DIR}/*.png"]}, {"cell_type": "code", "execution_count": 9, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["!find /kaggle > file_list.txt"]}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1"}}, "nbformat": 4, "nbformat_minor": 1}