{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About\n* 2020-NCAAM & 2020-NCAAW"},{"metadata":{},"cell_type":"markdown","source":"# Insights\n\n* dart is good accuracy, gdbt is not bad.\n* season backward=0 is best?, 3 is bad.\n\n## TODO & Changelog\n\n* Test backward=1,2\n  * backward=0 best\n* Add Public Ranking score(as ranking) to feature structure.\n  * added\n* Test dart,gbdt\n  * dart is good\n* Calibrate preds\n  * Add sig_clf = CalibratedClassifierCV(clf, method=\"sigmoid\", cv=\"prefit\")\n* Test ensamble\n  * Use VotingClassifier\n* Team Conferences with Conference Game in Men\n  * Done\n* GridSearch with VotingClassifier\n* Team Conferences in Women\n* Add Play-by-play to feature structure.\n* Inspect Team Coaches."},{"metadata":{},"cell_type":"markdown","source":"# Flags"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG=True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy import stats\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV, KFold, StratifiedKFold, train_test_split\nfrom sklearn.calibration import CalibratedClassifierCV, calibration_curve\nfrom sklearn.metrics import log_loss, brier_score_loss, plot_roc_curve\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn import preprocessing\nfrom sklearn.pipeline import Pipeline\nimport lightgbm as lgb\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Game result data class\n* Season: regular season data\n* Tourney: NCAA 64-68 teams per year"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DetailedResult():\n    def __init__(self, df):\n        self.raw = df\n\n    # location Home/Away/Neutral to int 1/0/-1\n    def __get_loc(self,loc,lose=False):\n        wloc_to_num = {'H':1, 'A': -1, 'N': 0}\n        if loc in wloc_to_num:\n            if lose:\n                return - wloc_to_num[loc]\n            else:\n                return wloc_to_num[loc]\n        else:\n            raise ValueError\n\n    # Wxx Lxx\n    def __get_neu_list(self,prefix=''):\n        neu_keys = ['TeamID', 'Score', 'Loc', 'FGM', 'FGA', 'FGM3', 'FGA3', 'FTM', 'FTA', 'OR', 'DR', 'Ast', 'TO', 'Stl', 'Blk', 'PF']\n        conv = list(map(lambda x: str(prefix) + x, neu_keys))\n        return conv\n\n    def __get_neu_dict(self,prefix=''):\n        neu_keys = self.__get_neu_list()\n        conv = list(map(lambda x: str(prefix) + x, neu_keys))\n        r = dict()\n        for n,c in zip(neu_keys,conv):\n            r = {**r,c:n}\n        return r\n\n    def __get_ren_dict(self,prefix1='',prefix2='T'):\n        neu_keys = self.__get_neu_list()\n        conv1 = list(map(lambda x: str(prefix1) + x, neu_keys))\n        conv2 = list(map(lambda x: str(prefix2) + x, neu_keys))\n        r = dict()\n        for c1,c2 in zip(conv1,conv2):\n            r = {**r,c1:c2}\n        return r\n\n    # expand from detail to average and std\n    def __get_ave_std_dict(self):\n        cols = self.__get_neu_list()\n        cols.remove('TeamID')\n        cols.append('NumOT')\n        d=dict()\n        for col in cols:\n            d = {**d, col: [np.average,np.std]}\n        return d\n    \n    def get_raw(self):\n        return self.raw\n\n    def __loc_norm(self,win_1st):\n        # Location flag for Lose team\n        win_1st['LLoc'] = win_1st['WLoc'].map(lambda x: self.__get_loc(x,lose=True))\n        win_1st['WLoc'] = win_1st['WLoc'].map(lambda x: self.__get_loc(x,lose=False))\n        return win_1st\n    \n    def get_match_result(self, season, backward=0):\n        \"\"\"\n        TeamID1,TeamID2,...,result(TeamID1) pair scores and features\n        \n        Parameters\n        ----------\n        season: int\n            Season\n        backword: int\n            go back years\n        \"\"\"\n        win_1st = self.raw.copy()\n        win_1st = win_1st.loc[(win_1st['Season'] <= season) & (win_1st['Season'] >= season-backward)].reset_index(drop=True)\n        \n        # Location flag for Lose team\n        win_1st = self.__loc_norm(win_1st)\n        \n        # copy lose_1st\n        lose_1st = win_1st.copy()\n        lose_1st.rename(columns=self.__get_ren_dict('W','T'), inplace=True)\n        lose_1st.rename(columns=self.__get_ren_dict('L','W'), inplace=True)\n        lose_1st.rename(columns=self.__get_ren_dict('T','L'), inplace=True)\n        \n        # Add result(1st team)\n        win_1st['result'] = 1\n        lose_1st['result'] = 0\n\n        # concat win and lose (So agg wiht 1Team only, 2Team is only use for mean)\n        merge = pd.concat((win_1st,lose_1st)).reset_index(drop=True)\n        _a = merge\n        \n        _a.rename(columns=self.__get_ren_dict('W','1'), inplace=True)\n        _a.rename(columns=self.__get_ren_dict('L','2'), inplace=True)\n\n        # concat\n        res = _a\n        \n        # feature assemble\n        res['1Score_diff'] = res['1Score'] - res['2Score']\n        res['1FG_rate_diff'] = res['1FGM'] / res['1FGA'] - res['2FGM'] / res['2FGA']\n        res['1FG3_rate_diff'] = res['1FGM3'] / res['1FGA3'] - res['2FGM3'] / res['2FGA3']\n        res['1FT_rate_diff'] = res['1FTM'] / res['1FTA'] - res['2FTM'] / res['2FTA']\n        res['1OR_diff'] = res['1OR'] - res['2OR']\n        res['1DR_diff'] = res['1DR'] - res['2DR']\n        res['1Ast_diff'] = res['1Ast'] - res['2Ast']\n        res['1TO_diff'] = res['1TO'] - res['2TO']\n        res['1Stl_diff'] = res['1Stl'] - res['2Stl']\n        res['1Blk_diff'] = res['1Blk'] - res['2Blk']\n        res['1PF_diff'] = res['1PF'] - res['2PF']\n        res = res.groupby(['1TeamID']).agg({'result': np.mean, # win_rate\n                                            'NumOT': np.mean,\n                                            '1Score': [np.mean,np.std],\n                                            '1Score_diff': [np.mean,np.std],\n                                            '1Loc': np.mean,\n                                            '1FG_rate_diff': np.mean,\n                                            '1FG3_rate_diff': np.mean,\n                                            '1FT_rate_diff': np.mean,\n                                            '1OR_diff':  np.mean,\n                                            '1DR_diff': np.mean,\n                                            '1Ast_diff':  np.mean,\n                                            '1TO_diff': np.mean,\n                                            '1Stl_diff':  np.mean,\n                                            '1Blk_diff': np.mean,\n                                            '1PF_diff': np.mean,\n                                           }) # mean with backward season included\n        res.columns = [\"_\".join(x) for x in res.columns.ravel()]\n        res = res.rename(columns={'result_mean':'1win_rate'})\n        res = res.reset_index()\n        return res\n\n    def get_feature_keys(self,prefix=''):\n        feature_keys = ['win_rate','Score_mean','Score_std','Score_diff_mean','Score_diff_std','Loc_mean',\n                        'FG_rate_diff_mean','FG3_rate_diff_mean','FT_rate_diff_mean','OR_diff_mean', 'DR_diff_mean',\n                        'Ast_diff_mean','TO_diff_mean','Stl_diff_mean','Blk_diff_mean','PF_diff_mean']\n        conv = list(map(lambda x: str(prefix) + x, feature_keys))\n        return conv\n    \n    def __get_feature_ren_dict(self,prefix1='',prefix2=''):\n        feature_keys = ['TeamID',*self.get_feature_keys()]\n        conv1 = list(map(lambda x: str(prefix1) + x, feature_keys))\n        conv2 = list(map(lambda x: str(prefix2) + x, feature_keys))\n        r = dict()\n        for c1,c2 in zip(conv1,conv2):\n            r = {**r,c1:c2}\n        return r\n    \n    def get_team_feature(self,season,backward=0,prefix=False):\n        res = self.get_match_result(season,backward)        \n        res = res[['1TeamID',*self.get_feature_keys('1')]]\n        res = res.sort_values(by=['1TeamID']).reset_index(drop=True)\n        res.rename(columns=self.__get_feature_ren_dict('1',prefix),inplace=True)\n        return res \n    \n    def get_as_pre_submission(self):\n        \"\"\"\n        get as Pre-Submission(TeamID1,TeamID2,result(truth)) format\n        \n        \"\"\"\n        res = self.raw.copy()\n        res['result'] = 1\n\n        # exchange TeamID wiht submission format TeamID1 < TeamID2\n        _a = res.loc[res['WTeamID'] < res['LTeamID']]\n        _a.rename(columns=self.__get_ren_dict('W','1'), inplace=True)\n        _a.rename(columns=self.__get_ren_dict('L','2'), inplace=True)\n        _b = res.loc[res['WTeamID'] > res['LTeamID']]\n        _b.rename(columns=self.__get_ren_dict('W','2'), inplace=True)\n        _b.rename(columns=self.__get_ren_dict('L','1'), inplace=True)\n\n        # result=0 if swaped\n        _b['result'] = 0\n        \n        res = pd.concat((_a,_b)).reset_index(drop=True)\n        res.rename(columns={'1TeamID':'TeamID1','2TeamID':'TeamID2'},inplace=True)\n        res = res[['Season','TeamID1','TeamID2','result']]\n        res = res.sort_values(by=['Season','TeamID1','TeamID2']).reset_index(drop=True)\n        \n        return res\n        \n    def get_score(self,prefix=''):\n        \"\"\"\n        Single Team scores (for exploration, not use now)\n        \"\"\"\n        tmp = self.raw.copy()\n        \n        # Location flag for Lose team\n        tmp = self.__loc_norm(tmp)\n\n        win = tmp[['Season','DayNum', 'NumOT', *self.__get_neu_list('W')]]\n        lose = tmp[['Season','DayNum', 'NumOT', *self.__get_neu_list('L')]]\n\n        win.rename(columns=self.__get_neu_dict('W'), inplace=True)\n        lose.rename(columns=self.__get_neu_dict('L'), inplace=True)\n        \n        win[prefix+'Result'] = 1\n        lose[prefix+'Result'] = 0\n\n        merge = pd.concat((win,lose)).reset_index(drop=True)\n        \n        # ave/std\n        merge = merge.groupby(['Season', 'TeamID'])\n        merge = merge.agg({'DayNum': 'count', prefix+'Result': np.average, **self.__get_ave_std_dict()}).rename(columns={'DayNum': prefix+'Count'})\n        \n        # rename column with prefix \n        merge.rename(columns=self.__get_ren_dict('', prefix), inplace=True)\n        \n        return merge\n\n    def get_st_score(self):\n        tmp = self.get_score()\n        tmp_group = tmp.groupby(['Season', 'TeamID'])\n        return tmp_group.agg({'DayNum': 'count', 'Result': np.average, **self.__get_ave_std_dict()}).rename(columns={'DayNum': 'Count'})\n\n# test\ndef runtest():\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyDetailedResults.csv')\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')\n    tr = DetailedResult(tourney_result)\n    sr = DetailedResult(season_result)\n    \n    assert tr.get_raw().shape == (1115, 34)\n    assert sr.get_raw().shape == (87504, 34)\n    \n    #return sr.get_team_feature(2019,3,'X')\n    assert sr.get_team_feature(2019,3,'X').shape == (353,17)\n    \n    #return tr.get_as_pre_submission()\n    assert tr.get_as_pre_submission().shape == (1115,4)\n    \n    #return tr.get_match_result(2019,1)\n    assert tr.get_match_result(2019,1).shape == (104,18)\n    #return tr.get_match_result(2019,1)\n    return sr.get_team_feature(2019,3,'X')\nif DEBUG:\n    runtest()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed data class\n1. 1985-2019\n2. 64-68 team per year"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Seed():\n    def __init__(self,df):\n        self.raw = df\n\n    # convert seed W01 to (int)1\n    def __get_seed(self,x):\n        return int(x[1:3])\n\n    def get_raw(self):\n        return self.raw\n\n    def get_seed(self,season=False):\n        tmp = self.raw.copy()\n        tmp['Seed'] = tmp['Seed'].map(lambda x: self.__get_seed(x))\n        if season:\n            tmp = tmp.loc[tmp['Season']==season]\n        return tmp\n        \n# test\ndef runtest1():\n    ts = Seed(tourney_seed)\n    ts = ts.get_seed()\n    sr = DetailedResult(season_result)\n    sr = sr.get_match_result(2019,1)\n    \n    a = pd.merge(ts, sr, left_on=['Season','TeamID'], right_on=['Season','1TeamID'], how='left')\n    \n    return a\n\ndef runtest2():\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\n    ts = Seed(tourney_seed)\n    return ts.get_seed()\n\nif DEBUG:    \n    runtest2()\n    #a.loc[(a['Season']==2018) & (a['Seed']==1)]\n    #a.loc[a['Season'] == 2019].groupby(['TeamID']).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Public ranking class\n* MMasseyOrdinals\n* RankingDayNum: 0-133\n  * Last RankingDayNum: 133\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MasseyOrdinals():\n    def __init__(self,df):\n        self.raw = df\n\n    def get_raw(self):\n        return self.raw\n    \n    def get_feature_keys(self):\n        l = list(self.get_team_feature().columns)\n        l.remove('Season')\n        l.remove('TeamID')\n        return l\n\n    def __get_systems(self):\n        return self.raw['SystemName'].unique()\n\n    def get_team_feature(self,season=False):\n        \"\"\"\n        (Season,TeamID, *SystemName)\n        (2019, 1111, 12,123,12,1)\n        \"\"\"\n        tmp = self.raw.copy()\n        tmp = tmp.loc[(tmp['RankingDayNum'] == 133)].drop('RankingDayNum',axis=1)\n        tmp = tmp.fillna(0)\n        tmp = tmp.groupby(['Season','TeamID','SystemName']).mean()        \n        tmp = tmp.unstack()\n        tmp.columns = tmp.columns.droplevel()\n        tmp = tmp.reset_index()\n\n        if season:\n            tmp = tmp.loc[tmp['Season'] == season]\n            #tmp = tmp.loc[(season)]\n        return tmp\n\ndef runtest():\n    mo_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MMasseyOrdinals.csv')\n    mo = MasseyOrdinals(mo_df)\n    #assert mo.get_raw().shape == (3820919,5)\n    #assert mo.get_team_feature().shape == (5833, 164)\n    #assert mo.get_team_feature(2010).shape == (347, 164)\n    #assert mo.get_team_feature(2014).shape == (351, 164)\n    return mo.get_team_feature(2014)\n\ndef runtest1():\n    mo_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MMasseyOrdinals.csv')\n    mo = MasseyOrdinals(mo_df)\n    return mo.get_feature_keys()\n\nif DEBUG:\n    a = runtest1()\n    #a = runtest()\n    #a.loc[(2010)]\n    a\n    #a.loc[a['SystemName'] == 'BBT'].sort_values(['OrdinalRank'])\n    #a.loc[a['TeamID'] == 1181].sort_values(['OrdinalRank'])\n    #a.loc[a['TeamID'] == 1181]\n\n    #df = a.loc[(2019,1101)] - a.loc[(2019,1102)]\n    #df.add_prefix('1').add_suffix('__diff')\n\n    #df = pd.merge(df1,df2,on=['Season'])\n    #df\n    #a.columns\n    #a.index\n    #a.add_prefix('1_')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Team Conference Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TeamConference():\n    def __init__(self,tc_df,ctg_df):\n        self.raw_tc = tc_df\n        self.raw_ctg = ctg_df\n\n    def get_feature_keys(self):\n        l = list(self.get_team_feature().columns)\n        l.remove('Season')\n        l.remove('TeamID')\n        return l\n    \n    def get_team_feature(self,season=False):\n        __tc = self.raw_tc.copy()\n        __tc = pd.get_dummies(__tc,dtype=np.int8)\n        if season:\n            __tc = __tc.loc[__tc['Season']==season]\n        __tmp = self.get_match_result(season)        \n        __tmp = pd.merge(__tc, __tmp, on=['Season','TeamID'],how='left')\n        #__tmp = __tmp.sort_values(by=['1TeamID']).reset_index(drop=True)\n        #__tmp.rename(columns=self.__get_feature_ren_dict('1',prefix),inplace=True)\n        return __tmp\n    \n    def get_match_result(self,season=False):\n        __tmp = pd.DataFrame()\n        win_1st = self.raw_ctg.rename(columns={'WTeamID':'TeamID'}).drop(['DayNum','LTeamID'],axis=1)\n        win_1st['result'] = 1\n        lose_1st = self.raw_ctg.rename(columns={'LTeamID':'TeamID'}).drop(['DayNum','WTeamID'],axis=1)\n        lose_1st['result'] = 0\n        __tmp = pd.concat((win_1st,lose_1st)).reset_index(drop=True)\n        if season:\n            __tmp = __tmp.loc[__tmp['Season']==season]\n        __tmp = __tmp.groupby(['Season','TeamID']).agg({'result': [np.mean,np.std]})\n        __tmp.columns = [\"_\".join(x) for x in __tmp.columns.ravel()]\n        return __tmp.reset_index()\n    \n    \ndef runtest():\n    tc_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MTeamConferences.csv')\n    ctg_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MConferenceTourneyGames.csv')\n    tc = TeamConference(tc_df,ctg_df)\n    assert tc.get_match_result(2007).shape == (294,4)\n    assert tc.get_team_feature(2007).shape == (336,55)\n    assert tc.get_team_feature(2012).shape == (345,55)\n    assert tc.get_team_feature(2017).shape == (351,55)\n    #return tc.get_team_feature(2019)\n    return tc.get_feature_keys()\n\nif DEBUG:\n    runtest()\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission data class\n1. Seed 68 team means that 67 pertterns match pairs per year.\n2. Stage1 2015-2019 seed teams, predict by history that 2003-2019 season games and 2003-2019 tourney games.\n3. Stage2 2020 seed teams, predict by history that 2015-2020 season games and 2015-2019 tourney games.\n   PRED results of 2020 seed menbers pair, FROM 2020 season games score and results AND 2019 tourny games.\n\nPRED results of Y seed menbers pair, FROM Y-12 to Y season games score and results AND Y-11 to Y-1 tourny games.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Submission():\n    \"\"\"\n    Submission class\n    \n    Fields\n    ------\n    ID : str\n        from csv field\n    Season : int\n        2019 etc\n    TeamID1 : int\n    TeamID2 : int\n    Pred : float\n        TeamID1 win rate\n    \"\"\"\n    def __init__(self,df):\n        self.raw = df\n\n    def __separate_id(self):\n        tmp = self.raw.copy()\n        tmp['Season'] = tmp['ID'].map(lambda x: int(x[:4]))\n        tmp['TeamID1'] = tmp['ID'].map(lambda x: int(x[5:9]))   # rate: Pred\n        tmp['TeamID2'] = tmp['ID'].map(lambda x: int(x[10:14])) # rate: 1-Pred\n        return tmp\n\n    def get_raw(self):\n        return self.raw\n    \n    def get_separated(self):\n        return self.__separate_id()\n    \n    def get_as_pre_submission(self):\n        \"\"\"\n        get as pre-submission format\n        \"\"\"\n        res = self.get_separated()\n        return res[['Season','TeamID1','TeamID2']]\n    \n    def get_with_result(self, detail_result):\n        \"\"\"\n        get df Join on Season,Temaid with result\n        \"\"\"\n        submit = self.get_separated()\n        submit = pd.merge(submit, detail_result.get_score('1'), left_on=['Season','TeamID1'], right_on=['Season', 'TeamID'], how='left')\n        submit = pd.merge(submit, detail_result.get_score('2'), left_on=['Season','TeamID2'], right_on=['Season', 'TeamID'], how='left')\n        submit = submit.drop(['ID','Pred','TeamID1','TeamID2'],axis=1)\n        return submit\n\n# test\ndef runtest1(): # with feature\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    sm = Submission(test_df)\n    sm = sm.get_separated()\n    sr = DetailedResult(season_result)\n    ts = Seed(tourney_seed)\n    \n    r = pd.DataFrame([])\n    for season in sm['Season'].unique():\n        _sm = sm.loc[sm['Season'] == season].reset_index(drop=True) # 1 season\n        _ts = ts.get_seed(season)\n        _sr1 = sr.get_team_feature(season,1,'1') # many season feature mean\n        _sr2 = sr.get_team_feature(season,1,'2') # many season feature mean\n        \n        tmp = pd.merge(_sm, _ts, left_on=['Season','TeamID1'], right_on=['Season','TeamID'], how='left').rename(columns={'Seed': 'Seed1'}).drop(['TeamID'],axis=1)\n        tmp = pd.merge(tmp, _ts, left_on=['Season','TeamID2'], right_on=['Season','TeamID'], how='left').rename(columns={'Seed': 'Seed2'}).drop(['TeamID'],axis=1)\n        tmp = pd.merge(tmp, _sr1, left_on=['TeamID1'], right_on=['1TeamID'], how='left')\n        tmp = pd.merge(tmp, _sr2, left_on=['TeamID2'], right_on=['2TeamID'], how='left')\n        \n        tmp.drop(['1TeamID','2TeamID'],axis=1)\n        tmp['1win_diff'] = tmp['1win_rate'] - tmp['2win_rate']\n        tmp['1score_diff'] = tmp['1score_mean'] - tmp['2score_mean']\n        tmp['1seed_diff'] = tmp['Seed1'] - tmp['Seed2']\n        r = r.append(tmp)\n    return r\n\ndef runtest2():\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    \n    s1 = Submission(test_df)\n    return s1.get_separated()\ndef runtest3():\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    \n    s1 = Submission(test_df)\n    assert s1.get_as_pre_submission().shape == (11390, 3)\n    assert s1.get_separated().shape == (11390, 5)\n    return s1.get_separated()\n    \nif DEBUG:    \n    a = runtest3()\n    a\n    #a.sort_values(by=['result'])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing train data\n1. TourneyResults(WTeamID,LTeamID,result) to pre-Submission(Season,TeamID1,TeamID2,result)\n2. Join Submission(TeamID1,TeamID2,result) in tourney_season=Y and SeasonFeatures(TeamID,features..) in season between Y and Y-backward as Team1 and Team2\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Prepare():\n    def __init__(self,\n                 tourney_result,\n                 season_result,\n                 tourney_seed,\n                 test_df,\n                 massey_ordinals=None,\n                 team_conference=None,\n                 conference_tourney_game=None\n                ):\n        self.backward = 0  # season feature backward\n        self.tr = DetailedResult(tourney_result)\n        self.sr = DetailedResult(season_result)\n        self.ts = Seed(tourney_seed)\n        self.s1 = Submission(test_df)\n        \n        if massey_ordinals is not None:\n            self.mo = MasseyOrdinals(massey_ordinals)\n        else:\n            self.mo = None\n\n        if (team_conference is not None) and (conference_tourney_game is not None):\n            self.tc = TeamConference(team_conference,conference_tourney_game)\n        else:\n            self.tc = None\n\n    def __prepare_df(self,ps):\n        r = pd.DataFrame()\n        for season in ps['Season'].unique():\n            _ps = ps.loc[ps['Season'] == season].reset_index(drop=True) # 1 season\n            _sr1 = self.sr.get_team_feature(season,self.backward,'1') # many season feature mean\n            _sr2 = self.sr.get_team_feature(season,self.backward,'2') # many season feature mean    \n            _ts = self.ts.get_seed(season)\n            \n            _tmp = pd.merge(_ps, _ts, left_on=['Season','TeamID1'], right_on=['Season','TeamID'], how='left').rename(columns={'Seed': 'Seed1'}).drop(['TeamID'],axis=1)\n            _tmp = pd.merge(_tmp, _ts, left_on=['Season','TeamID2'], right_on=['Season','TeamID'], how='left').rename(columns={'Seed': 'Seed2'}).drop(['TeamID'],axis=1)\n            _tmp = pd.merge(_tmp, _sr1, left_on=['TeamID1'], right_on=['1TeamID'], how='left').drop(['1TeamID'],axis=1)\n            _tmp = pd.merge(_tmp, _sr2, left_on=['TeamID2'], right_on=['2TeamID'], how='left').drop(['2TeamID'],axis=1)\n            if self.mo:\n                _mo = self.mo.get_team_feature(season)\n                _tmp = pd.merge(_tmp, _mo.add_prefix('1'), left_on=['Season','TeamID1'], right_on=['1Season','1TeamID'], how='left').drop(['1Season','1TeamID'],axis=1)\n                _tmp = pd.merge(_tmp, _mo.add_prefix('2'), left_on=['Season','TeamID2'], right_on=['2Season','2TeamID'], how='left').drop(['2Season','2TeamID'],axis=1)\n            if self.tc:\n                _tc = self.tc.get_team_feature(season)\n                _tmp = pd.merge(_tmp, _tc.add_prefix('1'), left_on=['Season','TeamID1'], right_on=['1Season','1TeamID'], how='left').drop(['1Season','1TeamID'],axis=1)\n                _tmp = pd.merge(_tmp, _tc.add_prefix('2'), left_on=['Season','TeamID2'], right_on=['2Season','2TeamID'], how='left').drop(['2Season','2TeamID'],axis=1)\n\n            # tourney feat diff\n            for ft in self.tr.get_feature_keys(): # + self.mo.get_feature_keys():\n                _tmp['1'+ft+'__diff'] = _tmp['1'+ft] - _tmp['2'+ft]\n                _tmp = _tmp.drop(['1'+ft,'2'+ft],axis=1)\n            \n            # massey ordinals day133 diff\n            if self.mo:\n                for ft in self.mo.get_feature_keys():\n                    _tmp['1'+ft+'__diff'] = _tmp['1'+ft] - _tmp['2'+ft]\n                    _tmp = _tmp.drop(['1'+ft,'2'+ft],axis=1)\n\n            # conference diff\n            if self.tc:\n                for ft in self.tc.get_feature_keys():\n                    _tmp['1'+ft+'__diff'] = _tmp['1'+ft] - _tmp['2'+ft]\n                    _tmp = _tmp.drop(['1'+ft,'2'+ft],axis=1)\n            \n            # seed diff\n            _tmp['1Seed__diff'] = _tmp['Seed1'] - _tmp['Seed2']\n            _tmp = _tmp.drop(['Seed1','Seed2'],axis=1)\n            \n            r = r.append(_tmp)\n\n        return r.sort_values(by=['Season','TeamID1','TeamID2']).reset_index(drop=True)\n\n    def __drop_list(self):\n        #return ['Season','TeamID1','TeamID2','1win_rate','2win_rate','1score_mean','2score_mean','Seed1','Seed2']\n        return ['Season','TeamID1','TeamID2']\n        \n    def get_pre_train(self):\n        ps = self.tr.get_as_pre_submission()\n        res = self.__prepare_df(ps)\n        X = res.drop('result',axis=1)\n        y = res.result\n        return X,y\n      \n    def get_train(self):\n        X,y = self.get_pre_train()\n        return X.drop(self.__drop_list(),axis=1), y\n    \n    def get_pre_test(self):\n        ps = self.s1.get_as_pre_submission()\n        X = self.__prepare_df(ps)\n        return X\n    \n    def get_test(self):\n        X = self.get_pre_test()\n        return X.drop(self.__drop_list(),axis=1)\n    \n\ndef runtest():\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyDetailedResults.csv')\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    massey_ordinals = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MMasseyOrdinals.csv')\n    tc_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MTeamConferences.csv')\n    ctg_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MConferenceTourneyGames.csv')\n\n    pre0 = Prepare(tourney_result,season_result,tourney_seed,test_df)\n    X,y = pre0.get_pre_train()\n    assert X.shape == (1115, 20)\n    assert y.shape == (1115,)\n    X,y = pre0.get_train()\n    assert X.shape == (1115, 17)\n    assert y.shape == (1115,)\n    X = pre0.get_test()\n    assert X.shape == (11390, 17)\n    #return X    \n    \n    pre = Prepare(tourney_result,season_result,tourney_seed,test_df,massey_ordinals)\n\n    X,y = pre.get_pre_train()\n    #return X\n    assert X.shape == (1115, 184)\n    assert y.shape == (1115,)\n    assert pre.get_pre_test().shape == (11390, 184)\n\n    X,y = pre.get_train()\n    #return X\n    assert X.shape == (1115, 181)\n    assert y.shape == (1115,)\n    \n    X = pre.get_test()\n    assert X.shape == (11390, 181)\n    #return X\n\n    \n    pre = Prepare(tourney_result,season_result,tourney_seed,test_df,None,tc_df,ctg_df)\n    X,y = pre.get_pre_train()\n    assert X.shape == (1115, 73)\n    X,y = pre.get_train()\n    assert X.shape == (1115, 70)\n    \n    pre = Prepare(tourney_result,season_result,tourney_seed,test_df,massey_ordinals,tc_df,ctg_df)\n    X,y = pre.get_pre_train()\n    assert X.shape == (1115, 237)\n    X,y = pre.get_train()\n    assert X.shape == (1115, 234)\n\nif DEBUG:\n    a =runtest()\n    #list(a.columns)\n    a","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load X,y,X_test"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_men():\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyDetailedResults.csv')\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    massey_ordinals = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MMasseyOrdinals.csv')\n    tc_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MTeamConferences.csv')\n    ctg_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MConferenceTourneyGames.csv')\n    pre = Prepare(tourney_result,season_result,tourney_seed,test_df,massey_ordinals,tc_df,ctg_df)\n    pre.backward = 0 # 3 season history backward not works.\n    X,y = pre.get_train()\n    X_test = pre.get_test()\n    return X,y,X_test\n\ndef load_women():\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneyDetailedResults.csv')\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonDetailedResults.csv')\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv')\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\n    #massey_ordinals = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WMasseyOrdinals.csv')\n    #tc_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WTeamConferences.csv')\n    #ctg_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WConferenceTourneyGames.csv')\n    pre = Prepare(tourney_result,season_result,tourney_seed,test_df)\n    pre.backward = 0 # 3 season history backward not works.\n    X,y = pre.get_train()\n    X_test = pre.get_test()\n    return X,y,X_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model tuning\n* load X,y,X_test from load_men() or load_women()\n* get best parameter with seach_params()"},{"metadata":{"trusted":true},"cell_type":"code","source":"def search_params(X,y):\n    X_train, X_eval, y_train, y_eval = train_test_split(X,y,test_size=0.2)\n    params = {\n        'device_type': 'cpu', # gpu (experimental)\n        'num_threads': 4,\n        'objective': 'binary',\n        'metric': ['binary_logloss'],\n\n        ### gbdt\n        #'early_stopping_rounds': 32, # 8  \n        #'num_iterations': 2000,    # 100 large\n        #'learning_rate': 1e-02, #stats.loguniform(1e-05, 3e-04),    # 0.1 accuracy/overfitting\n        #'boosting_type': 'gbdt', # gbdt(2e-03) dart(1e-02) # accuracy\n        #'max_depth': 3,\n        #'num_leaves': 5,\n\n        ### dart\n        'random_state': 0,\n        'num_iterations': 2000,    # 100 large\n        'learning_rate': 2e-02, # 0.1 small\n        'boosting_type': 'dart', # gbdt(2e-03) dart(1e-02)\n        'max_depth': 3,\n        'num_leaves': 5,\n        #'max_bin': 800, #stats.randint(255,2000),           # 255 500\n        \n        \n    }\n    grid_params = {   \n\n        ### dart\n        #'num_leaves': stats.randint(1,10), # 31 accuracy/overfitting 22\n        #'max_drop': [78], #stats.randint(40, 90), # 50 80\n        #'drop_rate': [0.06], #stats.uniform(0.0, 0.4), # 0.1 0.03\n        #'skip_drop': [0.117], #stats.uniform(0.0, 0.5), # 0.5\n\n        ### vs. overfitting like regularization\n\n        ## 1st tuning -- depend on #data\n        #'max_depth': stats.randint(1,4),  # -1\n        #'min_data_in_leaf': stats.randint(2,14), # 20 fixed of/uf\n        \n        ## 2nd tuning -- 2^(max_depth-1) < n < 2^(max_depth)\n        #'num_leaves': stats.randint(1,10), # 31 acc/of\n        \n        ## 3rd tuning -- accuracy\n        #'max_bin': stats.randint(250,1200), # 255 acc/of\n\n        ## 4th tuning\n        #'bagging_fraction': [0.785], #stats.uniform(0.5, 0.4), # 1.0 0.8\n        #'bagging_freq': [1], # 0\n        #'extra_trees': [True], # False\n\n        \n        ## 5th tuning\n        #'feature_fraction': stats.uniform(0.1, 0.9), # 1.0 like dropout\n        #'lambda_l1': stats.uniform(0.0, 1.0), # 0.0 0.85\n        #'lambda_l2': [0.49], #stats.uniform(0.2, 0.5), # 0.0 0.35\n        #'min_gain_to_split': stats.uniform(0.0, 0.5), # 0.0 0.25\n\n\n    }\n    clf = lgb.LGBMClassifier(**params)\n    gs = RandomizedSearchCV(\n        clf,\n        grid_params,\n        n_iter=10,\n        cv = KFold(n_splits=5, shuffle=True, random_state=params['random_state']),\n        scoring='neg_log_loss',\n        random_state=params['random_state'],\n        refit=True)\n    gs.fit(X_train, y_train,eval_set=[(X_eval,y_eval)],verbose=100)\n    \n    # plot logloss\n    lgb.plot_metric(gs.best_estimator_,'binary_logloss')\n\n    # calibration\n    iso_clf = CalibratedClassifierCV(gs.best_estimator_, method=\"isotonic\", cv='prefit')\n    iso_clf.fit(X_eval, y_eval)\n    sig_clf = CalibratedClassifierCV(gs.best_estimator_, method=\"sigmoid\", cv='prefit')\n    sig_clf.fit(X_eval, y_eval)\n    \n    # y_preds\n    y_preds_raw = gs.best_estimator_.predict_proba(X_eval)[:, 1]\n    y_preds_iso = iso_clf.predict_proba(X_eval)[:, 1]\n    y_preds_sig = sig_clf.predict_proba(X_eval)[:, 1]\n\n    print('Best -logloss score:\\t{:.4}'.format(gs.best_score_))\n    print('Best estimator score:\\t{:.4}'.format(log_loss(y_eval,y_preds_raw)))\n    print('Isotonic score:\\t\\t{:.4}'.format(log_loss(y_eval,y_preds_iso)))\n    print('Sigmoid  score:\\t\\t{:.4}'.format(log_loss(y_eval,y_preds_sig)))\n    \n    return gs.best_estimator_.get_params()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEBUG:\n    X,y,X_test = load_women()\n    param = search_params(X,y)\n    print(param)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculated params"},{"metadata":{"trusted":true},"cell_type":"code","source":"param_dart_women={\n    'boosting_type': 'dart',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.03,\n    'max_depth': 2,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 3,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    'num_iterations': 1000}\nparam_gbdt_women={\n    'boosting_type': 'gbdt',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.01,\n    'max_depth': 3,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 5,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    #'early_stopping_rounds': 32,\n    'num_iterations': 280,\n    'max_bin': 946}\nparam_dart_men={\n    'boosting_type': 'dart',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.03,\n    'max_depth': 2,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 3,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    'num_iterations': 400,\n    'max_bin': 800,\n    'bagging_fraction': 0.785,\n    'bagging_freq': 1,\n    'extra_trees': True,\n    'lambda_l2': 0.4971122519867125}\nparam_gbdt_men ={\n    'boosting_type': 'gbdt',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.01,\n    'max_depth': 2,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 3,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    #'early_stopping_rounds': 32,\n    'num_iterations': 2000}\n_param_gbdt_men ={\n    'boosting_type': 'gbdt',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.0003,\n    'max_depth': 3,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 31,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    #'early_stopping_rounds': 10,\n    'num_iterations': 15000, #10000\n    'max_bin': 1500,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 0,\n    'extra_trees': True,\n    'feature_fraction': 0.14942781480062634,\n    'lambda_l2': 0.6502883645891266,    \n    'min_data_in_leaf': 82}\n_param_gbdt_women ={\n    'boosting_type': 'gbdt',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.0003,\n    'max_depth': 3,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 31,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    #'early_stopping_rounds': 10,\n    'num_iterations': 15000, #11000\n    'max_bin': 1500,\n    'bagging_fraction': 0.85,\n    'bagging_freq': 0,    \n    'extra_trees': True,\n    'feature_fraction': 0.7141384275200073,\n    'lambda_l2': 0.2104430671327452,\n    'min_data_in_leaf': 36}\n_param_dart_men={\n    'boosting_type': 'dart',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.003,\n    'max_depth': 2,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 37,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    'num_iterations': 10000,\n    'bagging_fraction': 0.33,\n    'bagging_freq': 1,\n    'drop_rate': 0.084,\n    'extra_trees': True,\n    'feature_fraction': 0.10860473134218412,\n    'lambda_l1': 0.7489177872134295,\n    'lambda_l2': 0.4110745835653511,\n    'max_bin': 883,\n    'max_drop': 88,\n    'min_gain_to_split': 0.416577843839325,\n    'skip_drop': 0.35}\nparam_dart_women={\n    'boosting_type': 'dart',\n    'class_weight': None,\n    'colsample_bytree': 1.0,\n    'importance_type': 'split',\n    'learning_rate': 0.005,\n    'max_depth': 5,\n    'min_child_samples': 20,\n    'min_child_weight': 0.001,\n    'min_split_gain': 0.0,\n    'n_estimators': 100,\n    'n_jobs': -1,\n    'num_leaves': 31,\n    'objective': 'binary',\n    #'random_state': None,\n    'reg_alpha': 0.0,\n    'reg_lambda': 0.0,\n    'silent': True,\n    'subsample': 1.0,\n    'subsample_for_bin': 200000,\n    'subsample_freq': 0,\n    'device_type': 'cpu',\n    'num_threads': 4,\n    'metric': ['binary_logloss'],\n    'num_iterations': 12500,\n    'max_bin': 1600,\n    'bagging_fraction': 0.656,\n    'bagging_freq': 1,\n    'drop_rate': 0.06,\n    'extra_trees': True,\n    'feature_fraction': 0.5041555816486761,\n    'lambda_l1': 0.14348565828538462,\n    'lambda_l2': 0.03137365290350269,\n    'max_drop': 78,\n    'min_data_in_leaf': 23,\n    'min_gain_to_split': 0.29629331241595064,\n    'skip_drop': 0.117}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Voting function"},{"metadata":{"trusted":true},"cell_type":"code","source":" \ndef voting(X,y,gbdt_params=None,dart_params=None):\n    \n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n    log_losses = defaultdict(list)\n    \n    for train_index,eval_index in tqdm(list(skf.split(X,y))):\n        X_train, X_eval = X.iloc[train_index], X.iloc[eval_index]\n        y_train, y_eval = y[train_index], y[eval_index]\n        if gbdt_params is None:\n            gbdt_params = {    \n                'device_type': 'cpu', # gpu (experimental)\n                'num_threads': 4,\n                'objective': 'binary',\n                'metric': ['binary_logloss'],\n                'num_iterations': 25000,    # 100 large\n                'learning_rate': 3e-04, #stats.loguniform(1e-05, 3e-04),    # 0.1 accuracy/overfitting\n                'boosting_type': 'gbdt', # gbdt(2e-03) dart(1e-02) # accuracy\n                'max_bin': 1500, #stats.randint(1000,1800), # 255 accuracy/overfitting\n            }\n        if dart_params is None:\n            dart_params = {\n                'device_type': 'cpu', # gpu (experimental)\n                'num_threads': 4,\n                'objective': 'binary',\n                'metric': ['binary_logloss'],\n                'num_iterations': 4000,    # 100 large\n                'learning_rate': 3e-03, # 0.1 small\n                'boosting_type': 'dart', # gbdt(2e-03) dart(1e-02)\n                'max_bin': 255, #stats.randint(1000,2000),           # 255 500\n                'num_leaves': 22, #stats.randint(40,120), # 31 accuracy/overfitting 22\n                'max_drop': 80, # 50\n                'drop_rate': 0.03, #stats.uniform(0.0, 1.0), # 0.1\n                'skip_drop': 0.5, #stats.uniform(0.3, 0.4), # 0.5\n            }\n\n        estimators = [\n            ('lgb1', lgb.LGBMClassifier(**gbdt_params,random_state=0)),\n            ('lgb2', lgb.LGBMClassifier(**dart_params,random_state=1)),\n            ('lgb3', lgb.LGBMClassifier(**gbdt_params,random_state=2)),\n            ('lgb4', lgb.LGBMClassifier(**dart_params,random_state=3)),\n            ('lgb5', lgb.LGBMClassifier(**gbdt_params,random_state=4)),\n            ('lgb6', lgb.LGBMClassifier(**dart_params,random_state=5)),\n        ]\n\n        voting = VotingClassifier(estimators,voting='soft')\n        voting.fit(X_train,y_train)\n        \n        y_preds = voting.predict_proba(X_eval)\n        score = log_loss(y_eval,y_preds)\n        log_losses['voting'].append(score)\n\n        sig_voting = CalibratedClassifierCV(voting, method=\"sigmoid\", cv='prefit')\n        sig_voting.fit(X_eval, y_eval)\n\n        y_preds_sig = sig_voting.predict_proba(X_eval)\n        score_sig = log_loss(y_eval,y_preds_sig)\n        log_losses['sig_voting'].append(score_sig)\n        \n        for name, estimator in voting.named_estimators_.items():\n            y_preds = estimator.predict_proba(X_eval)\n            ll = log_loss(y_eval,y_preds)\n            log_losses[name].append(ll)\n        \n    for name, ll_list in log_losses.items():\n        mean_ll = np.array(ll_list).mean()\n        print(name, ':', mean_ll)\n\n    return sig_voting\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mens\n## Voting model train, predict and submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"X,y,X_test = load_men()\nsig_voting = voting(X,y,param_gbdt_men,param_dart_men)\n#sig_voting = voting(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds_sig = sig_voting.predict_proba(X_test)[:, 1]\ntest_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\ntest_df['Pred'] = y_preds_sig\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission_m.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Womens\n## Voting model train, predict and submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"X,y,X_test = load_women()\nsig_voting = voting(X,y,param_gbdt_women,param_dart_women)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds_sig = sig_voting.predict_proba(X_test)[:, 1]\ntest_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\ntest_df['Pred'] = y_preds_sig\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission_w.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}