{"cells":[{"metadata":{},"cell_type":"markdown","source":"    Based on this kernel:\n    https://www.kaggle.com/artgor/march-madness-2020-ncaam-eda-and-baseline\n    \n    Looking through some of the notebooks from previous years, I noticed people getting better scores by using a spline on their test predictions.  In the discussion\n    section people are also recommending you clip your predictions to avoid getting a large penalty when your model predicts too confidently a team that doesn't win.\n    \n    In this kernel I compare the base prediction scores with the scores where the predictions have been adjusted by the spline and optimal clips.  The spline and clips are fit \n    on a holdback set using the games from 2019 since the public leaderboard doesn't seem very useful in this competition.\n    \n    This is also my first real attempt at writing a class in python as a mostly self taught programmer so your comments/criticism/hate mail is welcome."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import log_loss, mean_absolute_error, mean_squared_error\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tourney_result = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyDetailedResults.csv')\ntourney_seed = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\nseason_result = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')\ntest_df = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_win_result = season_result[['Season', 'WTeamID', 'WScore', 'WFGM', 'WFGA', 'WFGM3', 'WFGA3', 'WFTM', 'WFTA', 'WOR', 'WDR',\n                                  'WAst', 'WTO', 'WStl', 'WBlk', 'WPF']]\nseason_lose_result = season_result[['Season', 'LTeamID', 'LScore', 'LFGM', 'LFGA', 'LFGM3', 'LFGA3', 'LFTM', 'LFTA', 'LOR', 'LDR',\n                                   'LAst', 'LTO', 'LStl', 'LBlk', 'LPF']]\nseason_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score', 'WFGM':'FGM', 'WFGA':'FGA', 'WFGM3':'FGM3', 'WFGA3':'FGA3',\n                                  'WFTM':'FTM', 'WFTA':'FTA', 'WOR':'OR', 'WDR':'DR', 'WAst':'Ast', 'WTO':'TO', 'WStl':'Stl',\n                                  'WBlk':'Blk', 'WPF':'PF'}, inplace=True)\nseason_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score', 'LFGM':'FGM', 'LFGA':'FGA', 'LFGM3':'FGM3', 'LFGA3':'FGA3',\n                                  'LFTM':'FTM', 'LFTA':'FTA', 'LOR':'OR', 'LDR':'DR', 'LAst':'Ast', 'LTO':'TO', 'LStl':'Stl',\n                                  'LBlk':'Blk', 'LPF':'PF'}, inplace=True)\nseason_result = pd.concat((season_win_result, season_lose_result)).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result['Score_difference'] = tourney_result['WScore'] - tourney_result['LScore']\n\ntourney_result = tourney_result[['Season', 'WTeamID', 'LTeamID', 'Score_difference']]\n\ntourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result['WSeed'] = tourney_result['WSeed'].apply(lambda x: int(x[1:3]))\ntourney_result['LSeed'] = tourney_result['LSeed'].apply(lambda x: int(x[1:3]))\nprint(tourney_result.info(null_counts=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))\ntest_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))\ntest_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))\n\ntest_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in season_result.columns[2:]:\n    season_result_map_mean = season_result.groupby(['Season', 'TeamID'])[col].mean().reset_index()\n\n    tourney_result = pd.merge(tourney_result, season_result_map_mean, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={f'{col}':f'W{col}MeanT'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n    tourney_result = pd.merge(tourney_result, season_result_map_mean, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={f'{col}':f'L{col}MeanT'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n        \n    test_df = pd.merge(test_df, season_result_map_mean, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={f'{col}':f'W{col}MeanT'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)\n    \n    test_df = pd.merge(test_df, season_result_map_mean, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={f'{col}':f'L{col}MeanT'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result = tourney_result.drop(['WTeamID', 'LTeamID'], axis=1)\nfor col in tourney_win_result.columns[2:]:\n    if col[0] == 'W':\n        tourney_win_result.rename(columns={f'{col}':f'{col[1:]+\"1\"}'}, inplace=True)\n    elif col[0] == 'L':\n        tourney_win_result.rename(columns={f'{col}':f'{col[1:]+\"2\"}'}, inplace=True)\n        \ntourney_lose_result = tourney_win_result.copy()\nfor col in tourney_lose_result.columns:\n    if col[-1] == '1':\n        col2 = col[:-1] + '2'\n        tourney_lose_result[col] = tourney_win_result[col2]\n        tourney_lose_result[col2] = tourney_win_result[col]\ntourney_lose_result.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']\ntourney_win_result['ScoreMeanT_diff'] = tourney_win_result['ScoreMeanT1'] - tourney_win_result['ScoreMeanT2']\n\ntourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']\ntourney_lose_result['ScoreMeanT_diff'] = tourney_lose_result['ScoreMeanT1'] - tourney_lose_result['ScoreMeanT2']\n\ntourney_lose_result['Score_difference'] = -tourney_lose_result['Score_difference']\ntourney_win_result['result'] = 1\ntourney_lose_result['result'] = 0\n\ntourney_result = pd.concat((tourney_win_result, tourney_lose_result)).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in test_df.columns[2:]:\n    if col[0] == 'W':\n        test_df.rename(columns={f'{col}':f'{col[1:]+\"1\"}'}, inplace=True)\n    elif col[0] == 'L':\n        test_df.rename(columns={f'{col}':f'{col[1:]+\"2\"}'}, inplace=True)\n        \ntest_df['Seed1'] = test_df['Seed1'].apply(lambda x: int(x[1:3]))\ntest_df['Seed2'] = test_df['Seed2'].apply(lambda x: int(x[1:3]))\ntest_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']\ntest_df['ScoreMeanT_diff'] = test_df['ScoreMeanT1'] - test_df['ScoreMeanT2']\ntest_df = test_df.drop(['ID', 'Pred', 'Season'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.optimize import minimize\nfrom scipy.interpolate import UnivariateSpline\nfrom functools import partial\n\ndef minimize_clipper(labels, preds, clips):\n    clipped = np.clip(preds, clips[0], clips[1])\n    return log_loss(labels, clipped)\n\ndef spline_model(labels, preds):\n    comb = pd.DataFrame({'labels':labels, 'preds':preds})\n    comb = comb.sort_values(by='preds').reset_index(drop=True)\n    spline_model = UnivariateSpline(comb['preds'].values, comb['labels'].values)\n    adjusted = spline_model(preds)\n    return spline_model, log_loss(labels, adjusted)\n\nclass NCAA_model():\n    \n    def __init__(self, params, train_df, test_df, use_holdback=True, regression=False, verbose=True):\n        self.params = params\n        self.verbose = verbose\n        self.test_df = test_df\n        self.has_trained_models = False\n        self.models = []\n        if use_holdback == True:\n            self.use_holdback=2019\n        else:\n            self.use_holdback = use_holdback\n            \n        if regression:\n            self.params['objective'] = 'regression'\n            self.params['metric'] = 'mae'\n            self.target = 'Score_difference'\n            self.eval_func = mean_absolute_error\n        else:\n            self.params['objective'] = 'binary'\n            self.params['metric'] = 'logloss'\n            self.target = 'result'\n            self.eval_func = log_loss\n            \n        if not self.verbose:\n            self.params['verbosity'] = -1 \n            \n        if self.use_holdback:\n            self.holdback_df = train_df.query(f'Season == {self.use_holdback}')\n            self.holdback_target = self.holdback_df[self.target]\n            self.train_df = train_df.query(f'Season != {self.use_holdback}')\n        else:\n            self.train_df = train_df\n            \n        self.target = self.train_df[self.target]\n        \n    def train(self, features, n_splits, n_boost_round=5000, early_stopping_rounds=None, verbose_eval=1000):\n        self.feature_importances = pd.DataFrame(columns=features)\n        self.preds = np.zeros(shape=(self.test_df.shape[0]))\n        self.train_preds = np.zeros(shape=self.train_df.shape[0])\n        self.oof = np.zeros(shape=(self.train_df.shape[0]))\n        if self.use_holdback:\n            self.holdback_preds = np.zeros(shape=(self.holdback_df.shape[0]))\n        \n        cv = KFold(n_splits=n_splits, random_state=0, shuffle=True)        \n        for fold, (tr_idx, v_idx) in enumerate(cv.split(self.train_df)):\n            if self.verbose:\n                print(f'Fold: {fold}')\n                \n            x_train, y_train = self.train_df.iloc[tr_idx][features], self.target.iloc[tr_idx]\n            x_valid, y_valid = self.train_df.iloc[v_idx][features], self.target.iloc[v_idx]\n            X_t = lgb.Dataset(x_train, y_train)\n            X_v = lgb.Dataset(x_valid, y_valid)\n            \n            if self.has_trained_models:\n                self.models[fold] = lgb.train(params, X_t, num_boost_round = n_boost_round, early_stopping_rounds=early_stopping_rounds,\n                                                  valid_sets = [X_t, X_v], verbose_eval=(verbose_eval if self.verbose else None),\n                                                                                        init_model=self.models[fold])                \n            else:\n                model = lgb.train(params, X_t, num_boost_round = n_boost_round, early_stopping_rounds=early_stopping_rounds,\n                                                  valid_sets = [X_t, X_v], verbose_eval=(verbose_eval if self.verbose else None))\n                self.models.append(model)\n                \n            self.oof[v_idx] = self.models[fold].predict(x_valid)\n            self.train_preds[tr_idx] += self.models[fold].predict(x_train) / (n_splits-1)\n            self.preds += self.models[fold].predict(self.test_df[features]) / n_splits\n            self.feature_importances[f'fold_{fold}'] = self.models[fold].feature_importance()\n            if self.use_holdback:\n                self.holdback_preds += self.models[fold].predict(self.holdback_df[features]) / n_splits\n            \n            \n        tr_score = self.eval_func(self.target, self.train_preds)\n        oof_score = self.eval_func(self.target, self.oof)\n        print(f'Training {self.params[\"metric\"]}: {tr_score}')\n        print(f'OOF {self.params[\"metric\"]}: {oof_score}')\n        self.has_trained_models = True\n        if self.use_holdback:\n            hb_score = self.eval_func(self.holdback_target, self.holdback_preds)\n            print(f'Holdback set {self.params[\"metric\"]}: {hb_score}')\n            return tr_score, oof_score, hb_score\n        return tr_score, oof_score\n        \n    def fit_clipper(self, verbose=True):\n        preds = self.holdback_preds if self.use_holdback else self.oof\n        conv_target = np.where(self.holdback_target>0,1,0) if self.use_holdback else np.where(self.target>0,1,0)\n\n        partial_func = partial(minimize_clipper, conv_target, preds)\n        opt = minimize(partial_func, x0=[0.08, 0.92], method='nelder-mead')\n        if verbose:\n            print(f'Clip score: {opt.fun}')\n        clips = opt.x\n        score = opt.fun\n        return clips, score\n    \n    def fit_spline_model(self, verbose=True):\n        preds = self.holdback_preds if self.use_holdback else self.oof\n        conv_target = np.where(self.holdback_target>0,1,0) if self.use_holdback else np.where(self.target>0,1,0)\n        spline, score = spline_model(conv_target, preds)\n        if verbose:\n            print(f'Spline score: {score}')\n\n        return spline, score\n    \n    def postprocess_preds(self, opt_tool, method='clip', use_data='test', return_preds=False):\n        pred_dict = {'test':self.preds, 'train':self.train_preds, 'hb':self.holdback_preds, 'oof':self.oof}\n        label_dict = {'test':None, 'train':self.target, 'hb':self.holdback_target, 'oof':self.target}       \n        \n        if method == 'spline':\n            adjusted_preds = opt_tool(pred_dict[use_data])\n        elif method == 'clip':\n            adjusted_preds = np.clip(pred_dict[use_data], opt_tool[0], opt_tool[1])\n            \n        if use_data == 'test':\n            return adjusted_preds\n        if return_preds:\n            return adjusted_preds, self.eval_func(label_dict[use_data], adjusted_preds)\n        return self.eval_func(label_dict[use_data], adjusted_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [x for x in tourney_result.columns if x not in ['result', 'Score_difference', 'Season']]\nparams = {'num_leaves': 400,\n          'min_child_weight': 0.034,\n          'feature_fraction': 0.379,\n          'bagging_fraction': 0.418,\n          'min_data_in_leaf': 106,\n          'max_depth': -1,\n          'learning_rate': 0.0068,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          'reg_alpha': 0.3899,\n          'reg_lambda': 0.648,\n          'random_state': 47,\n         }\n\nstep_size = 20\nsteps = 250\nboosting_rounds = [step_size*(x+1) for x in range(steps)]\ndef run_boost_round_test(boosting_rounds, step_size):\n    training_scores, oof_scores, holdback_scores = [], [], []\n    model = NCAA_model(params, tourney_result, test_df, use_holdback=[2019], regression=False, verbose=False)   \n    print(f'Training for {step_size*steps} rounds.')\n    for rounds in range(step_size,boosting_rounds+1,step_size):\n        print(f'{\"*\"*50}')\n        print(f'Rounds: {rounds}')\n        if model.use_holdback:\n            tr_score, oof_score, hb_score = model.train(features, n_splits=10, n_boost_round=step_size, early_stopping_rounds=None)\n        else:\n            tr_score, oof_score = model.train(features, n_splits=10, n_boost_round=step_size, early_stopping_rounds=None)\n        clips, clip_s = model.fit_clipper(verbose=True)\n        spline, spline_s = model.fit_spline_model(verbose=True)\n        \n        training_scores.append([tr_score, model.postprocess_preds(clips, use_data = 'train'), \n                               model.postprocess_preds(spline, use_data = 'train', method='spline')])\n        oof_scores.append([oof_score, model.postprocess_preds(clips, use_data = 'oof'),\n                          model.postprocess_preds(spline, use_data = 'oof', method='spline')])\n        holdback_scores.append([hb_score, model.postprocess_preds(clips, use_data = 'hb'),\n                               model.postprocess_preds(spline, use_data = 'hb', method='spline')])\n\n\n    return training_scores, oof_scores, holdback_scores, model, clips, spline\n\ntraining_scores, oof_scores, holdback_scores, model, clips, spline = run_boost_round_test(boosting_rounds[-1], step_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_scores, oof_scores, holdback_scores\nfig,ax = plt.subplots(nrows=1,ncols=3, figsize=(20,5), sharey=True, sharex=True)\nplot_df = pd.DataFrame(data=training_scores, columns=['Classifier', 'Clipped', 'Spline'], index=boosting_rounds)\nplot_df.plot(ax=ax[0], title='Training')\nplot_df = pd.DataFrame(data=oof_scores, columns=['Classifier', 'Clipped', 'Spline'], index=boosting_rounds)\nplot_df.plot(ax=ax[1], title='Out of Fold')\nplot_df = pd.DataFrame(data=holdback_scores, columns=['Classifier', 'Clipped', 'Spline'], index=boosting_rounds)\nplot_df.plot(ax=ax[2], title='Holdback')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds = model.postprocess_preds(spline, method='spline')\nsubmission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = y_preds\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.describe()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}