{"cells":[{"metadata":{},"cell_type":"markdown","source":"# What is this kernel?\nEven though starting from the great baseline kernels, our GBDT models tend to overfit  easily.  \nAs a one of solutions, we tried to ensemble with more another algorithm −−　TrueSkill, a rating system developped by Microsoft.\n\nAbout TrueSkill:  \n  https://trueskill.org/  \n  https://www.microsoft.com/en-us/research/project/trueskill-ranking-system/  \n  \nOur model is based on:  \n  https://www.kaggle.com/artgor/march-madness-2020-ncaam-eda-and-baseline  \n  https://www.kaggle.com/khoongweihao/ncaam2020-xgboost-lightgbm-k-fold-baseline  \n  https://www.kaggle.com/zeemeen/ncaa-trueskill-script  "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Libraries\nimport numpy as np\nimport pandas as pd\npd.set_option('max_columns', None)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('seaborn')\n%matplotlib inline\nimport copy\nimport datetime\nimport lightgbm as lgb\nfrom scipy import stats\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV, RepeatedStratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn import model_selection\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix\nimport json\nimport ast\nimport time\nfrom sklearn import linear_model\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport glob\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading the data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"data_dict = {}\nfor i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/*'):\n    name = i.split('/')[-1].split('.')[0]\n    if name != 'MTeamSpellings':\n        data_dict[name] = pd.read_csv(i)\n    else:\n        data_dict[name] = pd.read_csv(i, encoding='cp1252')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# process seed\ndata_dict['MNCAATourneySeeds']['Seed'] = data_dict['MNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))\n# take only useful columns\ndata_dict['MNCAATourneySeeds'] = data_dict['MNCAATourneySeeds'][['Season', 'TeamID', 'Seed']]\ndata_dict['MNCAATourneyCompactResults'] = data_dict['MNCAATourneyCompactResults'][['Season','WTeamID', 'LTeamID']]\n\n# merge the data and rename the columns\ndf = pd.merge(data_dict['MNCAATourneyCompactResults'], data_dict['MNCAATourneySeeds'],\n              how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ndf = pd.merge(df, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'])\ndf = df.drop(['TeamID_x', 'TeamID_y'], axis=1)\ndf.columns = ['Season', 'WTeamID', 'LTeamID', 'WSeed', 'LSeed']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"team_win_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'WTeamID']).agg({'WScore':['sum', 'count']}).reset_index()\nteam_win_score.columns = ['Season', 'WTeamID', 'WScore_sum', 'WScore_count']\nteam_loss_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'LTeamID']).agg({'LScore':['sum', 'count']}).reset_index()\nteam_loss_score.columns = ['Season', 'LTeamID', 'LScore_sum', 'LScore_count']\n\ndf = pd.merge(df, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ndf = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ndf = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ndf = pd.merge(df, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ndf.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\n\ndf['x_score'] = df['WScore_sum_x'] + df['LScore_sum_y']\ndf['y_score'] = df['WScore_sum_y'] + df['LScore_sum_x']\ndf['x_count'] = df['WScore_count_x'] + df['LScore_count_y']\ndf['y_count'] = df['WScore_count_y'] + df['WScore_count_x']\n\ndf = df.drop(['WScore_sum_x','WScore_count_x','LScore_sum_x','LScore_count_x',\n              'LScore_sum_y','LScore_count_y','WScore_sum_y','WScore_count_y'], axis =1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df_win = df.copy()\ndf_los = df.copy()\ndf_win = df_win[['WSeed', 'LSeed', 'x_score', 'y_score', 'x_count', 'y_count']]\ndf_los = df_los[['LSeed', 'WSeed', 'y_score', 'x_score', 'x_count', 'y_count']]\ndf_win.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']\ndf_los.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']\ndf_win['Seed_diff'] = df_win['Seed_1'] - df_win['Seed_2']\ndf_win['Score_diff'] = df_win['Score_1'] - df_win['Score_2']\ndf_los['Seed_diff'] = df_los['Seed_1'] - df_los['Seed_2']\ndf_los['Score_diff'] = df_los['Score_1'] - df_los['Score_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df_win['result'] = 1\ndf_los['result'] = 0\ndata = pd.concat((df_win, df_los)).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize the data\n\nplt.figure(figsize=(24, 12))\ntmp1 = data[(data['result']==1)]\ntmp0 = data[(data['result']==0)]\nvis_cols = [c for c in data.columns if c not in ['result']]\nfor idx, col in enumerate(vis_cols):\n    plt.subplot(3,  3, idx+1)\n    plt.hist(tmp1[col], bins=25, alpha=0.5, label='win')\n    plt.hist(tmp0[col], bins=25, alpha=0.5, label='lose')\n    plt.legend(loc='best')\n    plt.title(col)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# preparing test"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n\ntest = test.drop(['Pred'], axis=1)\ntest['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))\ntest['Team1'] = test['ID'].apply(lambda x: int(x.split('_')[1]))\ntest['Team2'] = test['ID'].apply(lambda x: int(x.split('_')[2]))\ntest = pd.merge(test, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'Team1'], right_on=['Season', 'TeamID'])\ntest = pd.merge(test, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'Team2'], right_on=['Season', 'TeamID'])\ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'Team1'], right_on=['Season', 'WTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'Team2'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'Team1'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'Team2'], right_on=['Season', 'WTeamID'])\ntest.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test['x_score'] = test['WScore_sum_x'] + test['LScore_sum_y']\ntest['y_score'] = test['WScore_sum_y'] + test['LScore_sum_x']\ntest['x_count'] = test['WScore_count_x'] + test['LScore_count_y']\ntest['y_count'] = test['WScore_count_y'] + test['WScore_count_x']\n\ntest = test[['Seed_x', 'Seed_y', 'x_score', 'y_score', 'x_count', 'y_count']]\ntest.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']\n\ntest['Seed_diff'] = test['Seed_1'] - test['Seed_2']\ntest['Score_diff'] = test['Score_1'] - test['Score_2']\n\ntest_df = test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## modeling LightGBM"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"params_lgb = {'num_leaves': 127,\n          'min_data_in_leaf': 10,\n          'objective': 'binary',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"metric\": 'logloss',\n          \"verbosity\": -1,\n          'random_state': 42,\n         }\nX = data.drop('result', axis=1)\ny = data['result']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import gc\n\nNFOLDS = 10\nfolds = KFold(n_splits=NFOLDS)\n\ncolumns = X.columns\nsplits = folds.split(X, y)\ny_preds_lgb = np.zeros(test_df.shape[0])\ny_train_lgb = np.zeros(X.shape[0])\ny_oof = np.zeros(X.shape[0])\n\nfeature_importances = pd.DataFrame()\nfeature_importances['feature'] = columns\n  \nfor fold_n, (train_index, valid_index) in enumerate(splits):\n    print('Fold:',fold_n+1)\n    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    \n    dtrain = lgb.Dataset(X_train, label=y_train)\n    dvalid = lgb.Dataset(X_valid, label=y_valid)\n\n    clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)\n    \n    feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()\n    \n    y_pred_valid = clf.predict(X_valid)\n    y_oof[valid_index] = y_pred_valid\n    \n    y_train_lgb += clf.predict(X) / NFOLDS\n    y_preds_lgb += clf.predict(test_df) / NFOLDS\n    \n    del X_train, X_valid, y_train, y_valid\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.hist(y_preds_lgb);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"When we submit this preds, the score would be around 0.26; probably over fitting.  \nTo make our preds more moderate, we tried to ensemble with TrueSkill"},{"metadata":{},"cell_type":"markdown","source":"# Trying to ensemble with TrueSkill"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nabout TrueSkill, thanks to:\nhttps://www.kaggle.com/zeemeen/ncaa-trueskill-script\n\"\"\"\nimport trueskill\nfrom trueskill import rate_1vs1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expose_and_clip(rating, env=None, minimum=0., maximum=50.):\n    env = env if env else trueskill.global_env()\n    return min(max(minimum, env.expose(rating)), maximum)\nenv = trueskill.TrueSkill()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df_compact = data_dict['MNCAATourneyCompactResults']\ndf_detailed = data_dict['MNCAATourneyDetailedResults']\nresults_merged  = pd.merge(df_compact, df_detailed)\n\nresults_merged.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#we added 4 factors\nresults_merged['WeFG%'] = (results_merged['WFGM']+0.5*results_merged['WFGM3']) / results_merged['WFGA']\nresults_merged['LeFG%'] = (results_merged['LFGM']+0.5*results_merged['LFGM3']) / results_merged['LFGA']\nresults_merged['WTO%'] = results_merged['WTO'] / (results_merged['WFGA']+0.44*results_merged['WFTA'] +results_merged['WTO'])\nresults_merged['LTO%'] = results_merged['LTO'] / (results_merged['LFGA']+0.44*results_merged['LFTA'] +results_merged['LTO'])\nresults_merged['WFTR%'] = results_merged['WFTA'] / results_merged['WFGA']\nresults_merged['LFTR%'] = results_merged['LFTA'] / results_merged['LFGA']\nresults_merged['WORB%'] = results_merged['WOR'] / results_merged['LDR']\nresults_merged['LORB%'] = results_merged['LOR'] / results_merged['WDR']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom trueskill import TrueSkill, Rating, rate_1vs1\n\nts = TrueSkill(draw_probability=0.01) # 0.01 is arbitary small number\n#beta = 25 / 6  # default value\nbeta = 25 / 6  \n\ndef win_probability(p1, p2):\n    delta_mu = p1.mu - p2.mu\n    sum_sigma = p1.sigma * p1.sigma + p2.sigma * p2.sigma\n    denom = np.sqrt(2 * (beta * beta) + sum_sigma)\n    return ts.cdf(delta_mu / denom)\n    \nsubmit = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmit[['Season', 'Team1', 'Team2']] = submit.apply(lambda r:pd.Series([int(t) for t in r.ID.split('_')]), axis=1)\n\ndf_tour = results_merged\nteamIds = np.unique(np.concatenate([df_tour.WTeamID.values, df_tour.LTeamID.values]))\nratings = { tid:ts.Rating() for tid in teamIds }\n\ndef feed_season_results(season):\n    print(\"season = {}\".format(season))\n    df1 = df_tour[df_tour.Season == season]\n    for r in df1.itertuples():\n        ratings[r.WTeamID], ratings[r.LTeamID] = rate_1vs1(ratings[r.WTeamID], ratings[r.LTeamID])\n\ndef update_pred(season):\n    beta = np.std([r.mu for r in ratings.values()]) \n    print(\"beta = {}\".format(beta))\n    submit.loc[submit.Season==season, 'Pred'] = submit[submit.Season==season].apply(lambda r:win_probability(ratings[r.Team1], ratings[r.Team2]), axis=1)\n\nfor season in sorted(df_tour.Season.unique())[:-5]: # exclude last 4 years\n    feed_season_results(season)\n\nupdate_pred(2015)\nfeed_season_results(2015)\nupdate_pred(2016)\nfeed_season_results(2016)\nupdate_pred(2017)\nfeed_season_results(2017)\nupdate_pred(2018)\nfeed_season_results(2018)\nupdate_pred(2019)\n\nsubmit.drop(['Season', 'Team1', 'Team2'], axis=1, inplace=True)\ny_preds_ts = submit['Pred']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submit['Pred']);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ensemble the predictions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = 0.4*y_preds_lgb + 0.6*y_preds_ts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission_df['Pred']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"When we only use LigntGBM, the score was around 0.26.  \nWhen we ensemble models with TrueSkill, the score was improved to around 0.35.  \nSo by using TrueSkill, we were able to make our model more moderate.  \n  \nAlthough it is still overfitting, but we can safely conclude that TrueSkill is a considerable choice for ensembling.  "}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}